2024-10-21 Mon 02:02:49:371.178 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: vhost interface name : vhost0 2024-10-21 Mon 02:02:49:371.337 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: vhost IP Address : 10.0.0.30/24 2024-10-21 Mon 02:02:49:371.347 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: vhost gateway : 0.0.0.0 2024-10-21 Mon 02:02:49:371.353 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Ethernet ports : ens3 2024-10-21 Mon 02:02:49:371.358 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Loopback IP : 0.0.0.0 2024-10-21 Mon 02:02:49:371.364 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp Servers : 10.0.0.30:5269 2024-10-21 Mon 02:02:49:371.371 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp Authentication : 1 2024-10-21 Mon 02:02:49:371.376 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2024-10-21 Mon 02:02:49:371.381 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2024-10-21 Mon 02:02:49:371.386 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2024-10-21 Mon 02:02:49:371.391 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Cluster-Name : 2024-10-21 Mon 02:02:49:371.398 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: DNS Servers : 10.0.0.30:53 2024-10-21 Mon 02:02:49:371.403 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: DNS client port : 8098 2024-10-21 Mon 02:02:49:371.408 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: DNS timeout : 3000 2024-10-21 Mon 02:02:49:371.427 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: DNS max retries : 2 2024-10-21 Mon 02:02:49:371.432 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp Dns Authentication : 1 2024-10-21 Mon 02:02:49:371.437 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2024-10-21 Mon 02:02:49:371.442 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2024-10-21 Mon 02:02:49:371.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2024-10-21 Mon 02:02:49:371.452 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: TSN Servers : [] 2024-10-21 Mon 02:02:49:371.457 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: COLLECTOR Servers : 10.0.0.30:8086 2024-10-21 Mon 02:02:49:371.462 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Tunnel-Type : MPLSoGRE 2024-10-21 Mon 02:02:49:371.467 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Metadata-Proxy Shared Secret: 3de2c9cd-4ea9-43a4-800e-2515b9e0af98 2024-10-21 Mon 02:02:49:371.472 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Metadata-Proxy Port : 8097 2024-10-21 Mon 02:02:49:371.477 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Metadata-Proxy SSL Flag : 0 2024-10-21 Mon 02:02:49:371.493 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Max Vm Flows : 100 2024-10-21 Mon 02:02:49:371.503 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Linklocal Max System Flows : 16384 2024-10-21 Mon 02:02:49:371.508 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Linklocal Max Vm Flows : 16384 2024-10-21 Mon 02:02:49:371.513 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow cache timeout : 0 2024-10-21 Mon 02:02:49:371.518 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Stale Interface cleanup timeout : 60 2024-10-21 Mon 02:02:49:371.523 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow thread count : 1 2024-10-21 Mon 02:02:49:371.527 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow latency limit : 0 2024-10-21 Mon 02:02:49:371.532 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow index-mgr sm log count : 0 2024-10-21 Mon 02:02:49:371.538 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow add-tokens : 50 2024-10-21 Mon 02:02:49:371.542 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow ksync-tokens : 25 2024-10-21 Mon 02:02:49:371.547 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow del-tokens : 16 2024-10-21 Mon 02:02:49:371.552 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow update-tokens : 16 2024-10-21 Mon 02:02:49:371.557 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Pin flow netlink task to CPU: 2024-10-21 Mon 02:02:49:371.562 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Maximum sessions : 100 2024-10-21 Mon 02:02:49:371.567 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Maximum session aggregates : 8 2024-10-21 Mon 02:02:49:371.572 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Maximum session endpoints : 5 2024-10-21 Mon 02:02:49:371.576 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Fabric SNAT hash table size : 4096 2024-10-21 Mon 02:02:49:371.581 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Flow excluding Router ID in hash :0 2024-10-21 Mon 02:02:49:371.586 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Agent Mode : Vrouter 2024-10-21 Mon 02:02:49:371.591 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Gateway Mode : None 2024-10-21 Mon 02:02:49:371.596 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: DHCP Relay Mode : 0 2024-10-21 Mon 02:02:49:371.601 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Service instance netns cmd : /usr/bin/opencontrail-vrouter-netns 2024-10-21 Mon 02:02:49:371.606 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Service instance docker cmd : /usr/bin/opencontrail-vrouter-docker 2024-10-21 Mon 02:02:49:371.611 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Service instance workers : 0 2024-10-21 Mon 02:02:49:371.616 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Service instance timeout : 0 2024-10-21 Mon 02:02:49:371.621 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Service instance lb ssl : 2024-10-21 Mon 02:02:49:371.625 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Service instance lbaas auth : 2024-10-21 Mon 02:02:49:371.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Bgp as a service port range : 50000-50512 2024-10-21 Mon 02:02:49:371.639 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Services queue limit : 1024 2024-10-21 Mon 02:02:49:371.644 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: BGPAAS max shared sessions for service port : 4 2024-10-21 Mon 02:02:49:371.649 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Sandesh Key file : /etc/contrail/ssl/private/server-privkey.pem 2024-10-21 Mon 02:02:49:371.654 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Sandesh Cert file : /etc/contrail/ssl/certs/server.pem 2024-10-21 Mon 02:02:49:371.658 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Sandesh CA Cert : /etc/contrail/ssl/certs/ca-cert.pem 2024-10-21 Mon 02:02:49:371.663 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Sandesh SSL Enable : 1 2024-10-21 Mon 02:02:49:371.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Introspect SSL Enable : 1 2024-10-21 Mon 02:02:49:371.673 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Hypervisor mode : kvm 2024-10-21 Mon 02:02:49:371.781 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Ethernet Port Encap Type : 2024-10-21 Mon 02:02:49:371.790 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: Platform mode : Vrouter on host linux kernel 2024-10-21 Mon 02:02:49:372.045 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Logging: DISABLED -> ENABLED 2024-10-21 Mon 02:02:49:372.945 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Logging: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] log4level: [ NOTSET ] -> [ DEBUG ] 2024-10-21 Mon 02:02:49:401.370 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:49:401.410 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> self 2024-10-21 Mon 02:02:49:401.418 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:49:401.424 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2024-10-21 Mon 02:02:49:401.430 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:49:401.435 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-access-control-list -> self 2024-10-21 Mon 02:02:49:401.441 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap access-control-list : self -> virtual-network-access-control-list 2024-10-21 Mon 02:02:49:401.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap access-control-list : virtual-network-access-control-list -> nil 2024-10-21 Mon 02:02:49:401.453 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:49:401.458 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> self 2024-10-21 Mon 02:02:49:401.463 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:49:401.475 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> virtual-network-network-ipam 2024-10-21 Mon 02:02:49:401.481 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:49:401.487 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:49:401.492 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-qos-config -> self 2024-10-21 Mon 02:02:49:401.498 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap qos-config : self -> virtual-network-qos-config 2024-10-21 Mon 02:02:49:401.503 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap qos-config : virtual-network-qos-config -> nil 2024-10-21 Mon 02:02:49:401.509 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:49:401.514 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-security-logging-object -> self 2024-10-21 Mon 02:02:49:401.520 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : self -> virtual-network-security-logging-object 2024-10-21 Mon 02:02:49:401.525 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : virtual-network-security-logging-object -> nil 2024-10-21 Mon 02:02:49:401.531 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:49:401.536 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : logical-router-virtual-network -> self 2024-10-21 Mon 02:02:49:401.542 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-router-virtual-network : self -> logical-router-virtual-network 2024-10-21 Mon 02:02:49:401.547 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-router-virtual-network : logical-router-virtual-network -> logical-router-virtual-network 2024-10-21 Mon 02:02:49:401.621 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-router : self -> logical-router-virtual-network 2024-10-21 Mon 02:02:49:401.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-router : logical-router-virtual-network -> nil 2024-10-21 Mon 02:02:49:401.637 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:49:401.643 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-multicast-policy -> self 2024-10-21 Mon 02:02:49:401.649 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap multicast-policy : self -> virtual-network-multicast-policy 2024-10-21 Mon 02:02:49:401.654 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap multicast-policy : virtual-network-multicast-policy -> nil 2024-10-21 Mon 02:02:49:401.661 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for routing-instance 2024-10-21 Mon 02:02:49:401.670 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> self 2024-10-21 Mon 02:02:49:401.676 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:49:401.682 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-routing-instance 2024-10-21 Mon 02:02:49:401.687 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2024-10-21 Mon 02:02:49:401.692 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2024-10-21 Mon 02:02:49:402.100 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for security-group 2024-10-21 Mon 02:02:49:402.114 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-group : security-group-access-control-list -> self 2024-10-21 Mon 02:02:49:402.120 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap access-control-list : self -> security-group-access-control-list 2024-10-21 Mon 02:02:49:402.126 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap access-control-list : security-group-access-control-list -> nil 2024-10-21 Mon 02:02:49:402.135 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for tag 2024-10-21 Mon 02:02:49:402.140 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : application-policy-set-tag -> self 2024-10-21 Mon 02:02:49:402.147 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:49:402.152 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : policy-management-application-policy-set -> application-policy-set-tag 2024-10-21 Mon 02:02:49:402.159 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap policy-management : policy-management-application-policy-set -> nil 2024-10-21 Mon 02:02:49:402.165 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.172 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2024-10-21 Mon 02:02:49:402.177 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:402.183 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:402.188 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:49:402.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:49:402.201 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.211 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2024-10-21 Mon 02:02:49:402.217 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:49:402.222 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine : virtual-machine-interface-virtual-machine -> nil 2024-10-21 Mon 02:02:49:402.228 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.233 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2024-10-21 Mon 02:02:49:402.238 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2024-10-21 Mon 02:02:49:402.243 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> nil 2024-10-21 Mon 02:02:49:402.249 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.254 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2024-10-21 Mon 02:02:49:402.259 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:402.264 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:402.270 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2024-10-21 Mon 02:02:49:402.275 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2024-10-21 Mon 02:02:49:402.281 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.286 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-routing-instance -> self 2024-10-21 Mon 02:02:49:402.292 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface-routing-instance : self -> virtual-machine-interface-routing-instance 2024-10-21 Mon 02:02:49:402.297 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface-routing-instance : virtual-machine-interface-routing-instance -> virtual-machine-interface-routing-instance 2024-10-21 Mon 02:02:49:402.303 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : self -> virtual-machine-interface-routing-instance 2024-10-21 Mon 02:02:49:402.308 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : virtual-machine-interface-routing-instance -> nil 2024-10-21 Mon 02:02:49:402.314 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.322 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-group -> self 2024-10-21 Mon 02:02:49:402.327 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-group : self -> virtual-machine-interface-security-group 2024-10-21 Mon 02:02:49:402.333 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-group : virtual-machine-interface-security-group -> nil 2024-10-21 Mon 02:02:49:402.339 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.344 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : alias-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.350 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap alias-ip : self -> alias-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.355 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap alias-ip : alias-ip-pool-alias-ip -> alias-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.361 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap alias-ip-pool : virtual-network-alias-ip-pool -> alias-ip-pool-alias-ip 2024-10-21 Mon 02:02:49:402.367 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-alias-ip-pool 2024-10-21 Mon 02:02:49:402.372 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-alias-ip-pool -> nil 2024-10-21 Mon 02:02:49:402.380 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.384 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.390 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.395 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2024-10-21 Mon 02:02:49:402.407 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:49:402.412 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:49:402.417 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2024-10-21 Mon 02:02:49:402.423 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2024-10-21 Mon 02:02:49:402.428 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.442 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.452 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2024-10-21 Mon 02:02:49:402.472 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:49:402.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:49:402.483 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:49:402.488 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2024-10-21 Mon 02:02:49:402.494 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.499 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.504 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.509 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap floating-ip : instance-ip-floating-ip -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.514 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap instance-ip : instance-ip-virtual-network -> instance-ip-floating-ip 2024-10-21 Mon 02:02:49:402.519 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> instance-ip-virtual-network 2024-10-21 Mon 02:02:49:402.525 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : instance-ip-virtual-network -> nil 2024-10-21 Mon 02:02:49:402.530 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.535 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : instance-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.541 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap instance-ip : self -> instance-ip-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.546 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap instance-ip : instance-ip-virtual-machine-interface -> nil 2024-10-21 Mon 02:02:49:402.552 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.562 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-route-table -> self 2024-10-21 Mon 02:02:49:402.572 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap interface-route-table : self -> virtual-machine-interface-route-table 2024-10-21 Mon 02:02:49:402.577 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap interface-route-table : virtual-machine-interface-route-table -> nil 2024-10-21 Mon 02:02:49:402.583 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.588 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : subnet-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.594 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap subnet : self -> subnet-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.600 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap subnet : subnet-virtual-machine-interface -> nil 2024-10-21 Mon 02:02:49:402.607 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.612 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.619 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> logical-interface-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.625 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2024-10-21 Mon 02:02:49:402.905 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:49:402.915 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:49:402.923 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.928 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-port-group-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.943 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> virtual-port-group-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.949 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : physical-router-physical-interface -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:49:402.954 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:49:402.959 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:49:402.966 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:402.972 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:402.982 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.988 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bgp-as-a-service : bgpaas-bgp-router -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:49:402.994 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bgp-router : self -> bgpaas-bgp-router 2024-10-21 Mon 02:02:49:402.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bgp-router : instance-bgp-router -> bgpaas-bgp-router 2024-10-21 Mon 02:02:49:403.005 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : self -> instance-bgp-router 2024-10-21 Mon 02:02:49:403.010 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : instance-bgp-router -> nil 2024-10-21 Mon 02:02:49:403.016 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.021 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:403.027 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:49:403.032 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bgp-as-a-service : bgpaas-health-check -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:49:403.053 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap service-health-check : self -> bgpaas-health-check 2024-10-21 Mon 02:02:49:403.058 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap service-health-check : bgpaas-health-check -> nil 2024-10-21 Mon 02:02:49:403.064 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.069 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-qos-config -> self 2024-10-21 Mon 02:02:49:403.075 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap qos-config : self -> virtual-machine-interface-qos-config 2024-10-21 Mon 02:02:49:403.081 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap qos-config : virtual-machine-interface-qos-config -> nil 2024-10-21 Mon 02:02:49:403.087 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.091 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-logging-object -> self 2024-10-21 Mon 02:02:49:403.097 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : self -> virtual-machine-interface-security-logging-object 2024-10-21 Mon 02:02:49:403.102 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : virtual-machine-interface-security-logging-object -> nil 2024-10-21 Mon 02:02:49:403.109 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.118 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2024-10-21 Mon 02:02:49:403.124 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:49:403.129 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:49:403.136 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:49:403.141 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:49:403.146 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:49:403.152 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2024-10-21 Mon 02:02:49:403.157 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.162 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2024-10-21 Mon 02:02:49:403.167 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:49:403.172 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:49:403.178 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:49:403.183 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bridge-domain : virtual-machine-interface-bridge-domain -> nil 2024-10-21 Mon 02:02:49:403.190 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.195 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-tag -> self 2024-10-21 Mon 02:02:49:403.200 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : self -> virtual-machine-interface-tag 2024-10-21 Mon 02:02:49:403.206 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-interface-tag 2024-10-21 Mon 02:02:49:403.211 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.216 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.222 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.230 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.237 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.242 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:49:403.249 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.254 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2024-10-21 Mon 02:02:49:403.260 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:403.265 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-tag -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:403.270 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : self -> virtual-network-tag 2024-10-21 Mon 02:02:49:403.276 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : application-policy-set-tag -> virtual-network-tag 2024-10-21 Mon 02:02:49:403.281 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.286 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.291 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.295 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.301 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.306 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:49:403.312 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.317 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2024-10-21 Mon 02:02:49:403.322 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:49:403.328 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:49:403.336 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : self -> virtual-machine-tag 2024-10-21 Mon 02:02:49:403.341 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2024-10-21 Mon 02:02:49:403.346 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.351 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.396 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.408 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.413 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:49:403.420 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.425 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2024-10-21 Mon 02:02:49:403.431 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2024-10-21 Mon 02:02:49:403.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> virtual-machine-interface-sub-interface 2024-10-21 Mon 02:02:49:403.441 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:49:403.446 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:49:403.451 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : self -> virtual-machine-tag 2024-10-21 Mon 02:02:49:403.456 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2024-10-21 Mon 02:02:49:403.461 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.466 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.471 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.476 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.485 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.490 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:49:403.497 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.502 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : project-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:403.509 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap project : self -> project-virtual-machine-interface 2024-10-21 Mon 02:02:49:403.514 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap project : project-tag -> project-virtual-machine-interface 2024-10-21 Mon 02:02:49:403.519 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : self -> project-tag 2024-10-21 Mon 02:02:49:403.524 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : application-policy-set-tag -> project-tag 2024-10-21 Mon 02:02:49:403.529 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.534 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:49:403.539 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.544 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.549 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:403.554 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:49:403.560 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.565 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> self 2024-10-21 Mon 02:02:49:403.571 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2024-10-21 Mon 02:02:49:403.577 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2024-10-21 Mon 02:02:49:403.582 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2024-10-21 Mon 02:02:49:403.587 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> nil 2024-10-21 Mon 02:02:49:403.593 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:49:403.602 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : logical-router-interface -> self 2024-10-21 Mon 02:02:49:403.607 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-router : self -> logical-router-interface 2024-10-21 Mon 02:02:49:403.613 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-router : logical-router-interface -> nil 2024-10-21 Mon 02:02:49:403.619 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for physical-interface 2024-10-21 Mon 02:02:49:403.624 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : physical-router-physical-interface -> self 2024-10-21 Mon 02:02:49:403.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:49:403.635 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:49:403.640 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for physical-interface 2024-10-21 Mon 02:02:49:403.645 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : virtual-port-group-physical-interface -> self 2024-10-21 Mon 02:02:49:403.651 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-port-group-physical-interface : self -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:49:403.656 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-port-group-physical-interface : virtual-port-group-physical-interface -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:49:403.662 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-port-group : self -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:49:403.667 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> nil 2024-10-21 Mon 02:02:49:403.673 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for logical-interface 2024-10-21 Mon 02:02:49:403.678 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> self 2024-10-21 Mon 02:02:49:403.683 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : self -> physical-interface-logical-interface 2024-10-21 Mon 02:02:49:403.727 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2024-10-21 Mon 02:02:49:403.738 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:49:403.745 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:49:403.753 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for logical-interface 2024-10-21 Mon 02:02:49:403.759 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-interface : logical-interface-virtual-machine-interface -> self 2024-10-21 Mon 02:02:49:403.769 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : self -> logical-interface-virtual-machine-interface 2024-10-21 Mon 02:02:49:403.775 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> nil 2024-10-21 Mon 02:02:49:403.781 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for logical-interface 2024-10-21 Mon 02:02:49:403.786 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap logical-interface : physical-router-logical-interface -> self 2024-10-21 Mon 02:02:49:403.792 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : self -> physical-router-logical-interface 2024-10-21 Mon 02:02:49:403.797 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : physical-router-logical-interface -> nil 2024-10-21 Mon 02:02:49:403.803 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for physical-router 2024-10-21 Mon 02:02:49:403.808 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-router : physical-router-physical-interface -> self 2024-10-21 Mon 02:02:49:403.814 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:49:403.819 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap physical-interface : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:49:403.826 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for service-health-check 2024-10-21 Mon 02:02:49:403.831 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap service-health-check : service-port-health-check -> self 2024-10-21 Mon 02:02:49:403.837 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2024-10-21 Mon 02:02:49:404.016 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> service-port-health-check 2024-10-21 Mon 02:02:49:404.026 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:404.032 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:49:404.038 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:49:404.046 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for service-health-check 2024-10-21 Mon 02:02:49:404.051 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap service-health-check : service-port-health-check -> self 2024-10-21 Mon 02:02:49:404.056 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2024-10-21 Mon 02:02:49:404.066 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> service-port-health-check 2024-10-21 Mon 02:02:49:404.071 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2024-10-21 Mon 02:02:49:404.076 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2024-10-21 Mon 02:02:49:404.082 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2024-10-21 Mon 02:02:49:404.087 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> port-tuple-interface 2024-10-21 Mon 02:02:49:404.092 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:49:404.097 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:49:404.103 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:49:404.110 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for qos-config 2024-10-21 Mon 02:02:49:404.115 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap qos-config : global-qos-config-qos-config -> self 2024-10-21 Mon 02:02:49:404.122 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap global-qos-config : global-qos-config-qos-config -> nil 2024-10-21 Mon 02:02:49:404.131 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for forwarding-class 2024-10-21 Mon 02:02:49:404.136 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap forwarding-class : forwarding-class-qos-queue -> self 2024-10-21 Mon 02:02:49:404.142 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap qos-queue : self -> forwarding-class-qos-queue 2024-10-21 Mon 02:02:49:404.147 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap qos-queue : forwarding-class-qos-queue -> nil 2024-10-21 Mon 02:02:49:404.153 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:49:404.158 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : security-logging-object-network-policy -> self 2024-10-21 Mon 02:02:49:404.170 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap network-policy : self -> security-logging-object-network-policy 2024-10-21 Mon 02:02:49:404.175 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap network-policy : security-logging-object-network-policy -> nil 2024-10-21 Mon 02:02:49:404.181 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:49:404.186 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : security-logging-object-security-group -> self 2024-10-21 Mon 02:02:49:404.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-group : self -> security-logging-object-security-group 2024-10-21 Mon 02:02:49:404.199 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-group : security-logging-object-security-group -> nil 2024-10-21 Mon 02:02:49:404.205 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:49:404.210 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : firewall-policy-security-logging-object -> self 2024-10-21 Mon 02:02:49:404.217 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-security-logging-object : self -> firewall-policy-security-logging-object 2024-10-21 Mon 02:02:49:404.222 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-security-logging-object : firewall-policy-security-logging-object -> firewall-policy-security-logging-object 2024-10-21 Mon 02:02:49:404.228 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : firewall-policy-security-logging-object -> nil 2024-10-21 Mon 02:02:49:404.234 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:49:404.239 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap security-logging-object : firewall-rule-security-logging-object -> self 2024-10-21 Mon 02:02:49:404.245 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule-security-logging-object : self -> firewall-rule-security-logging-object 2024-10-21 Mon 02:02:49:404.250 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule-security-logging-object : firewall-rule-security-logging-object -> firewall-rule-security-logging-object 2024-10-21 Mon 02:02:49:404.256 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : firewall-rule-security-logging-object -> nil 2024-10-21 Mon 02:02:49:404.270 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for bridge-domain 2024-10-21 Mon 02:02:49:404.275 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2024-10-21 Mon 02:02:49:404.281 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:49:404.286 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:49:404.292 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:49:404.297 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2024-10-21 Mon 02:02:49:404.302 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for bridge-domain 2024-10-21 Mon 02:02:49:404.307 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2024-10-21 Mon 02:02:49:404.312 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:49:404.321 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2024-10-21 Mon 02:02:49:404.327 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for application-policy-set 2024-10-21 Mon 02:02:49:404.332 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> self 2024-10-21 Mon 02:02:49:404.338 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:404.343 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:404.348 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:49:404.353 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:49:404.361 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:49:404.366 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:49:404.372 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.377 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.392 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : firewall-rule-service-group -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.409 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap service-group : self -> firewall-rule-service-group 2024-10-21 Mon 02:02:49:404.414 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap service-group : firewall-rule-service-group -> nil 2024-10-21 Mon 02:02:49:404.420 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:49:404.425 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:49:404.431 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.613 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.633 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : firewall-rule-address-group -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.643 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap address-group : self -> firewall-rule-address-group 2024-10-21 Mon 02:02:49:404.649 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap address-group : address-group-tag -> firewall-rule-address-group 2024-10-21 Mon 02:02:49:404.655 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : self -> address-group-tag 2024-10-21 Mon 02:02:49:404.661 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : address-group-tag -> nil 2024-10-21 Mon 02:02:49:404.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:49:404.674 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:49:404.679 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.684 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.689 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.695 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : firewall-rule-tag -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.700 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : self -> firewall-rule-tag 2024-10-21 Mon 02:02:49:404.705 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap tag : firewall-rule-tag -> nil 2024-10-21 Mon 02:02:49:404.711 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:49:404.716 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:49:404.721 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.726 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.731 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:49:404.736 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Adding ReactorMap firewall-rule : firewall-policy-firewall-rule -> nil 2024-10-21 Mon 02:02:49:593.522 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 10.0.0.30:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2024-10-21 Mon 02:02:49:595.382 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Current receive sock buffer size is 262144 2024-10-21 Mon 02:02:49:595.513 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703255746304, Pid 1093848]: KsyncTxQueue CPU pinning policy <>. KsyncTxQueuen not pinned to CPU 2024-10-21 Mon 02:02:49:598.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Vrouter family is 34 2024-10-21 Mon 02:02:49:598.880 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Flow table size : 20185088 2024-10-21 Mon 02:02:49:598.953 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Flow table size : 161218560 2024-10-21 Mon 02:02:49:598.967 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SetHugePages: Bridge table size:20185088 Flow table size:161218560 2024-10-21 Mon 02:02:49:598.989 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Mem mapping hugepage file:/dev/hugepages2M/bridge size:20185088 2024-10-21 Mon 02:02:49:600.643 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Mem mapped hugepage file:/dev/hugepages2M/bridge to addr:0x7ff805400000 2024-10-21 Mon 02:02:49:600.682 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Mem mapping hugepage file:/dev/hugepages2M/flow size:161218560 2024-10-21 Mon 02:02:49:600.691 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Mem mapped hugepage file:/dev/hugepages2M/flow to addr:0x7ff7fba00000 2024-10-21 Mon 02:02:49:600.707 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Sending Huge Page configuration to VROUTER 2024-10-21 Mon 02:02:49:660.538 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Huge pages set successfully 2024-10-21 Mon 02:02:49:680.130 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Flow table size : 161218560 2024-10-21 Mon 02:02:49:680.296 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Flow table size : 20185088 2024-10-21 Mon 02:02:49:753.125 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: ROLE : Generator 2024-10-21 Mon 02:02:49:753.166 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: MODULE : contrail-vrouter-agent 2024-10-21 Mon 02:02:49:753.172 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: SOURCE : cn-jenkins-deploy-platform-juju-os-439-1 2024-10-21 Mon 02:02:49:753.177 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: NODE TYPE : Compute 2024-10-21 Mon 02:02:49:753.183 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: INSTANCE ID : 0 2024-10-21 Mon 02:02:49:753.188 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: HTTP SERVER PORT : 8085 2024-10-21 Mon 02:02:49:753.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: System Log Send Rate Limit: 0 -> 0 2024-10-21 Mon 02:02:49:755.176 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: HTTP Introspect Init 2024-10-21 Mon 02:02:49:755.422 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:8085 Initialization complete src/contrail-common/io/tcp_server.cc 137 2024-10-21 Mon 02:02:49:755.484 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Sandesh Http Server Port: 8085 2024-10-21 Mon 02:02:49:755.565 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: NOT Writing http_port 8085TO : /tmp/contrail-vrouter-agent.1092974.http_port 2024-10-21 Mon 02:02:49:755.573 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: CONNECT TO COLLECTOR: 1 2024-10-21 Mon 02:02:49:755.686 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Idle 2024-10-21 Mon 02:02:49:757.148 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: No Client: 1729476169755805 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1093848 http_port = 8085 start_time = 1729476169755698 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:49:757.531 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvStart in state Idle 2024-10-21 Mon 02:02:49:759.839 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Disconnect 2024-10-21 Mon 02:02:49:759.988 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvCollectorUpdate in state Disconnect 2024-10-21 Mon 02:02:49:760.287 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:02:49:760.347 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:760.383 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: SANDESH: Send FAILED: 1729476169759812 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2024-10-21 Mon 02:02:49:760.393 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:49:760.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:761.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface lo flags 65609 2024-10-21 Mon 02:02:49:762.031 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface ens3 flags 69699 2024-10-21 Mon 02:02:49:762.066 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface fan-252 flags 69699 2024-10-21 Mon 02:02:49:762.072 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface ftun0 flags 69699 2024-10-21 Mon 02:02:49:762.106 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface lxdbr0 flags 4099 2024-10-21 Mon 02:02:49:762.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vethceed3bb2 flags 69699 2024-10-21 Mon 02:02:49:762.142 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vethff94287d flags 69699 2024-10-21 Mon 02:02:49:762.147 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface veth567cfff1 flags 69699 2024-10-21 Mon 02:02:49:762.178 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vethbac30548 flags 69699 2024-10-21 Mon 02:02:49:762.183 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface veth402290ab flags 69699 2024-10-21 Mon 02:02:49:762.213 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface veth4cf19a6f flags 69699 2024-10-21 Mon 02:02:49:762.219 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vethc2ae4048 flags 69699 2024-10-21 Mon 02:02:49:762.248 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vethc1b275bc flags 69699 2024-10-21 Mon 02:02:49:762.253 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vethe9446158 flags 69699 2024-10-21 Mon 02:02:49:762.281 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface veth712d04e2 flags 69699 2024-10-21 Mon 02:02:49:762.286 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vethd92a2e9b flags 69699 2024-10-21 Mon 02:02:49:762.310 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface veth64775acc flags 69699 2024-10-21 Mon 02:02:49:762.316 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface docker0 flags 4099 2024-10-21 Mon 02:02:49:762.335 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface pkt1 flags 65601 2024-10-21 Mon 02:02:49:762.340 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface pkt3 flags 65601 2024-10-21 Mon 02:02:49:762.345 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface pkt2 flags 65601 2024-10-21 Mon 02:02:49:762.360 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface vhost0 flags 69699 2024-10-21 Mon 02:02:49:762.365 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink interface message RTM_NEWLINK for interface pkt0 flags 69699 2024-10-21 Mon 02:02:49:762.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface lo 2024-10-21 Mon 02:02:49:762.416 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface fan-252 2024-10-21 Mon 02:02:49:762.426 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface lxdbr0 2024-10-21 Mon 02:02:49:762.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface docker0 2024-10-21 Mon 02:02:49:762.446 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface vhost0 2024-10-21 Mon 02:02:49:762.459 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface lo 2024-10-21 Mon 02:02:49:762.469 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface fan-252 2024-10-21 Mon 02:02:49:762.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface ftun0 2024-10-21 Mon 02:02:49:762.489 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface vhost0 2024-10-21 Mon 02:02:49:762.499 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink address message RTM_NEWADDR for interface pkt0 2024-10-21 Mon 02:02:49:762.543 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2024-10-21 Mon 02:02:49:762.560 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.570 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.581 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 10.0.0.1/32 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.591 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 10.0.0.2/32 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.603 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 10.0.0.4/32 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.614 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 10.25.16.0/24 Interface lxdbr0 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.620 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2024-10-21 Mon 02:02:49:762.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 172.17.0.0/16 Interface docker0 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.644 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Handle netlink route message RTM_NEWROUTE : 252.0.0.0/8 Interface fan-252 GW 0.0.0.0 2024-10-21 Mon 02:02:49:762.649 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:49:762.654 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:49:762.659 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:49:762.664 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:49:762.669 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:49:762.674 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:49:762.679 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:49:762.684 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:49:762.689 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:49:762.695 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:49:762.700 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:49:762.710 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:49:762.715 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:49:762.720 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:49:762.725 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:49:762.730 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:49:762.735 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2024-10-21 Mon 02:02:49:762.740 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2024-10-21 Mon 02:02:49:762.745 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2024-10-21 Mon 02:02:49:762.750 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2024-10-21 Mon 02:02:49:762.755 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2024-10-21 Mon 02:02:49:762.761 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:49:762.769 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:49:762.774 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:49:762.779 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:49:762.783 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:49:764.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: SANDESH: Send FAILED: 1729476169759839 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:__default__ index = 0 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2024-10-21 Mon 02:02:49:764.444 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2024-10-21 Mon 02:02:49:764.468 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:764.484 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: SANDESH: Send FAILED: 1729476169759885 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:ip-fabric index = 1 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2024-10-21 Mon 02:02:49:764.490 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2024-10-21 Mon 02:02:49:764.498 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:764.623 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: SANDESH: Send FAILED: 1729476169759974 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1093848 http_port = 8085 start_time = 1729476169755698 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:49:764.645 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:49:764.658 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:764.729 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: SANDESH: Send FAILED: 1729476169760029 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1093848 http_port = 8085 start_time = 1729476169755698 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:49:764.745 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:49:764.754 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:764.778 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: SANDESH: Send FAILED: 1729476169760045 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2024-10-21 Mon 02:02:49:764.785 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:49:764.795 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:764.856 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: SANDESH: Send FAILED: 1729476169760338 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1093848 http_port = 8085 start_time = 1729476169755698 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:49:764.869 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:49:783.890 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:9091 Initialization complete src/contrail-common/io/tcp_server.cc 137 2024-10-21 Mon 02:02:49:784.149 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Router ID configured. Connection to controller initiated 2024-10-21 Mon 02:02:49:790.075 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:790.132 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476169789966 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:49:790.145 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:49:792.343 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:792.398 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476169790064 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.30, Collector connection down, No Configuration for self Number of connections:2, Expected: 3 ], ] ] ] 2024-10-21 Mon 02:02:49:792.408 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:49:792.418 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:792.431 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476169790622 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:49:792.438 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:49:792.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:49:792.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476169790655 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.30, XMPP:dns-server:10.0.0.30, Collector connection down, No Configuration for self ], ] ] ] 2024-10-21 Mon 02:02:49:792.493 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:49:793.411 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: TCP [SYS_ERR]: TcpServerMessageLog: Server TCP bind(fe80::f816:3eff:fec4:d814%vhost0:8097): Cannot assign requested address src/contrail-common/io/tcp_server.cc 111 2024-10-21 Mon 02:02:50:336.588 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: VRouter [SYS_INFO]: VersionInfoLog: {"build-info":[{"build-version":"nightly","build-time":"2024-10-21 00:13:21.399567","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-6068-1.novalocal","build-id":"nightly-nightly.el7","build-number":"@contrail"}]} src/contrail-common/base/misc_utils.cc 41 2024-10-21 Mon 02:02:50:337.804 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:338.419 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170337767 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:338.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:338.445 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:338.459 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170337980 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect event = Connect Event local_port = 33333 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:338.465 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:338.472 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:338.482 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170337999 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:338.487 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:339.724 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:339.761 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: SANDESH: Send FAILED: 1729476170339673 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect event = Connect Event local_port = 35221 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:339.779 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:343.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:33333::10.0.0.30:8093 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2024-10-21 Mon 02:02:50:344.029 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.30 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-10-21 Mon 02:02:50:345.543 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:345.582 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: SANDESH: Send FAILED: 1729476170345485 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 33333 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:345.601 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:348.078 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:35221::10.0.0.30:5269 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2024-10-21 Mon 02:02:50:348.103 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.30 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-10-21 Mon 02:02:50:348.200 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703243151104, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:348.237 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703243151104, Pid 1093848]: SANDESH: Send FAILED: 1729476170348176 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 35221 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:348.247 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703243151104, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:350.556 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:350.588 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: SANDESH: Send FAILED: 1729476170350542 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:350.597 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:350.614 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:350.626 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: SANDESH: Send FAILED: 1729476170350609 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:350.638 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:350.666 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:350.678 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: SANDESH: Send FAILED: 1729476170350661 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:350.684 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:350.684 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703243151104, Pid 1093848]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:50:350.762 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:350.768 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1/dns:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:50:350.773 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: SANDESH: Send FAILED: 1729476170350758 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:350.817 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:351.664 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:351.681 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703259944704, Pid 1093848]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1/dns:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:50:351.689 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: SANDESH: Send FAILED: 1729476170351653 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-dns@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:351.837 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703251547904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:352.040 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703259944704, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:352.062 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703259944704, Pid 1093848]: SANDESH: Send FAILED: 1729476170352027 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-control@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:352.076 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703259944704, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:352.044 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703247349504, Pid 1093848]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:50:363.449 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV success controller/src/xmpp/xmpp_connection.cc 402 2024-10-21 Mon 02:02:50:365.260 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:365.303 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170365238 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:365.313 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:366.566 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:366.591 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170366545 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:366.599 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:366.672 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:366.698 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170366633 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.30, Collector connection down, No Configuration for self ], ] ] ] 2024-10-21 Mon 02:02:50:366.705 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:50:367.798 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV success controller/src/xmpp/xmpp_connection.cc 402 2024-10-21 Mon 02:02:50:368.093 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:368.129 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170368076 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:368.137 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:369.416 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:370.342 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476170369388 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:50:370.359 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:50:370.372 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:374.167 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476170369478 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2024-10-21 Mon 02:02:50:374.240 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:50:513.060 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703259944704, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:514.697 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703259944704, Pid 1093848]: SANDESH: Send FAILED: 1729476170512943 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:02:50:514.725 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703259944704, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:50:523.479 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:50:523.539 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170523431 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:02:50:523.555 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:50:760.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:02:50:828.613 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:02:50:828.646 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:02:50:828.813 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Idle 2024-10-21 Mon 02:02:50:828.912 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:02:50:828.958 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170828683 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:02:50:828.994 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:50:829.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:02:50:829.013 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:02:50:829.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: SANDESH: Send FAILED: 1729476170828899 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1093848 http_port = 8085 start_time = 1729476169755698 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 31, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:50:829.199 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703272539904, Pid 1093848]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:50:829.243 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703417260352, Pid 1093848]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2024-10-21 Mon 02:02:50:933.077 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:02:50:991.477 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: SANDESH: Send FAILED: 1729476170933052 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Addition uuid = 1e338ece-fda8-4842-aa92-7ce58d0af31f name = default-domain:default-project:ip-fabric acl_uuid = 561b7c53-af63-41b3-9c25-ec39072a7243 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2024-10-21 Mon 02:02:50:991.514 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Wrong state: Idle for event: EvSandeshSend message: VnObjectLog 2024-10-21 Mon 02:02:50:991.527 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:02:50:991.553 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: SANDESH: Send FAILED: 1729476170939188 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 1e338ece-fda8-4842-aa92-7ce58d0af31f name = default-domain:default-project:ip-fabric acl_uuid = 561b7c53-af63-41b3-9c25-ec39072a7243 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2024-10-21 Mon 02:02:50:991.559 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703264143104, Pid 1093848]: Wrong state: Idle for event: EvSandeshSend message: VnObjectLog 2024-10-21 Mon 02:02:51:038.619 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:02:51:038.675 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476171038597 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 1e338ece-fda8-4842-aa92-7ce58d0af31f name = default-domain:default-project:ip-fabric acl_uuid = 561b7c53-af63-41b3-9c25-ec39072a7243 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2024-10-21 Mon 02:02:51:038.687 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Idle for event: EvSandeshSend message: VnObjectLog 2024-10-21 Mon 02:02:51:829.014 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:02:51:829.290 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:02:51:829.405 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:51:829.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476171829095 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:02:51:829.445 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:51:829.454 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:51:829.641 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: SANDESH: Send FAILED: 1729476171829392 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1093848 http_port = 8085 start_time = 1729476169755698 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 36, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:51:829.665 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140703268341504, Pid 1093848]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:57:821.817 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: vhost interface name : vhost0 2024-10-21 Mon 02:02:57:821.925 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: vhost IP Address : 10.0.0.30/24 2024-10-21 Mon 02:02:57:821.947 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: vhost gateway : 0.0.0.0 2024-10-21 Mon 02:02:57:821.960 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Ethernet ports : ens3 2024-10-21 Mon 02:02:57:821.966 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Loopback IP : 0.0.0.0 2024-10-21 Mon 02:02:57:821.972 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp Servers : 10.0.0.30:5269 2024-10-21 Mon 02:02:57:821.979 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp Authentication : 1 2024-10-21 Mon 02:02:57:821.984 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2024-10-21 Mon 02:02:57:821.990 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2024-10-21 Mon 02:02:57:821.995 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2024-10-21 Mon 02:02:57:822.000 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Cluster-Name : 2024-10-21 Mon 02:02:57:822.007 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: DNS Servers : 10.0.0.30:53 2024-10-21 Mon 02:02:57:822.012 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: DNS client port : 8098 2024-10-21 Mon 02:02:57:822.017 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: DNS timeout : 3000 2024-10-21 Mon 02:02:57:822.022 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: DNS max retries : 2 2024-10-21 Mon 02:02:57:822.027 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp Dns Authentication : 1 2024-10-21 Mon 02:02:57:822.032 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2024-10-21 Mon 02:02:57:822.037 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2024-10-21 Mon 02:02:57:822.042 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2024-10-21 Mon 02:02:57:822.048 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TSN Servers : [] 2024-10-21 Mon 02:02:57:822.053 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: COLLECTOR Servers : 10.0.0.30:8086 2024-10-21 Mon 02:02:57:822.058 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Tunnel-Type : MPLSoGRE 2024-10-21 Mon 02:02:57:822.063 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Metadata-Proxy Shared Secret: 3de2c9cd-4ea9-43a4-800e-2515b9e0af98 2024-10-21 Mon 02:02:57:822.068 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Metadata-Proxy Port : 8097 2024-10-21 Mon 02:02:57:822.072 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Metadata-Proxy SSL Flag : 0 2024-10-21 Mon 02:02:57:822.086 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Max Vm Flows : 100 2024-10-21 Mon 02:02:57:822.092 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Linklocal Max System Flows : 16384 2024-10-21 Mon 02:02:57:822.096 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Linklocal Max Vm Flows : 16384 2024-10-21 Mon 02:02:57:822.104 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow cache timeout : 0 2024-10-21 Mon 02:02:57:822.109 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Stale Interface cleanup timeout : 60 2024-10-21 Mon 02:02:57:822.114 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow thread count : 1 2024-10-21 Mon 02:02:57:822.119 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow latency limit : 0 2024-10-21 Mon 02:02:57:822.125 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow index-mgr sm log count : 0 2024-10-21 Mon 02:02:57:822.130 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow add-tokens : 50 2024-10-21 Mon 02:02:57:822.135 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow ksync-tokens : 25 2024-10-21 Mon 02:02:57:822.140 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow del-tokens : 16 2024-10-21 Mon 02:02:57:822.145 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow update-tokens : 16 2024-10-21 Mon 02:02:57:822.149 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Pin flow netlink task to CPU: 2024-10-21 Mon 02:02:57:822.155 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Maximum sessions : 100 2024-10-21 Mon 02:02:57:822.160 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Maximum session aggregates : 8 2024-10-21 Mon 02:02:57:822.164 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Maximum session endpoints : 5 2024-10-21 Mon 02:02:57:822.169 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Fabric SNAT hash table size : 4096 2024-10-21 Mon 02:02:57:822.175 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Flow excluding Router ID in hash :0 2024-10-21 Mon 02:02:57:822.179 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Agent Mode : Vrouter 2024-10-21 Mon 02:02:57:822.184 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Gateway Mode : None 2024-10-21 Mon 02:02:57:822.189 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: DHCP Relay Mode : 0 2024-10-21 Mon 02:02:57:822.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Service instance netns cmd : /usr/bin/opencontrail-vrouter-netns 2024-10-21 Mon 02:02:57:822.199 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Service instance docker cmd : /usr/bin/opencontrail-vrouter-docker 2024-10-21 Mon 02:02:57:822.204 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Service instance workers : 0 2024-10-21 Mon 02:02:57:822.209 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Service instance timeout : 0 2024-10-21 Mon 02:02:57:822.214 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Service instance lb ssl : 2024-10-21 Mon 02:02:57:822.219 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Service instance lbaas auth : 2024-10-21 Mon 02:02:57:822.224 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Bgp as a service port range : 50000-50512 2024-10-21 Mon 02:02:57:822.229 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Services queue limit : 1024 2024-10-21 Mon 02:02:57:822.234 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: BGPAAS max shared sessions for service port : 4 2024-10-21 Mon 02:02:57:822.242 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Sandesh Key file : /etc/contrail/ssl/private/server-privkey.pem 2024-10-21 Mon 02:02:57:822.247 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Sandesh Cert file : /etc/contrail/ssl/certs/server.pem 2024-10-21 Mon 02:02:57:822.251 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Sandesh CA Cert : /etc/contrail/ssl/certs/ca-cert.pem 2024-10-21 Mon 02:02:57:822.256 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Sandesh SSL Enable : 1 2024-10-21 Mon 02:02:57:822.261 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Introspect SSL Enable : 1 2024-10-21 Mon 02:02:57:822.266 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Hypervisor mode : kvm 2024-10-21 Mon 02:02:57:822.365 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Ethernet Port Encap Type : 2024-10-21 Mon 02:02:57:822.373 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: Platform mode : Vrouter on host linux kernel 2024-10-21 Mon 02:02:57:823.969 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Logging: DISABLED -> ENABLED 2024-10-21 Mon 02:02:57:824.037 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Logging: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] log4level: [ NOTSET ] -> [ DEBUG ] 2024-10-21 Mon 02:02:57:830.872 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:57:830.899 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> self 2024-10-21 Mon 02:02:57:830.906 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:57:830.912 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2024-10-21 Mon 02:02:57:830.919 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:57:830.924 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-access-control-list -> self 2024-10-21 Mon 02:02:57:830.944 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap access-control-list : self -> virtual-network-access-control-list 2024-10-21 Mon 02:02:57:830.950 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap access-control-list : virtual-network-access-control-list -> nil 2024-10-21 Mon 02:02:57:830.956 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:57:830.961 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> self 2024-10-21 Mon 02:02:57:830.967 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:57:830.972 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> virtual-network-network-ipam 2024-10-21 Mon 02:02:57:830.987 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:57:830.993 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:57:830.998 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-qos-config -> self 2024-10-21 Mon 02:02:57:831.005 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap qos-config : self -> virtual-network-qos-config 2024-10-21 Mon 02:02:57:831.010 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap qos-config : virtual-network-qos-config -> nil 2024-10-21 Mon 02:02:57:831.016 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:57:831.021 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-security-logging-object -> self 2024-10-21 Mon 02:02:57:831.027 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : self -> virtual-network-security-logging-object 2024-10-21 Mon 02:02:57:831.032 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : virtual-network-security-logging-object -> nil 2024-10-21 Mon 02:02:57:831.038 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:57:831.043 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : logical-router-virtual-network -> self 2024-10-21 Mon 02:02:57:831.049 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-router-virtual-network : self -> logical-router-virtual-network 2024-10-21 Mon 02:02:57:831.054 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-router-virtual-network : logical-router-virtual-network -> logical-router-virtual-network 2024-10-21 Mon 02:02:57:831.060 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-router : self -> logical-router-virtual-network 2024-10-21 Mon 02:02:57:831.065 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-router : logical-router-virtual-network -> nil 2024-10-21 Mon 02:02:57:831.071 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-network 2024-10-21 Mon 02:02:57:831.076 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-multicast-policy -> self 2024-10-21 Mon 02:02:57:831.081 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap multicast-policy : self -> virtual-network-multicast-policy 2024-10-21 Mon 02:02:57:831.086 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap multicast-policy : virtual-network-multicast-policy -> nil 2024-10-21 Mon 02:02:57:831.092 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for routing-instance 2024-10-21 Mon 02:02:57:831.098 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> self 2024-10-21 Mon 02:02:57:831.103 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:57:831.111 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-routing-instance 2024-10-21 Mon 02:02:57:831.116 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2024-10-21 Mon 02:02:57:831.122 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2024-10-21 Mon 02:02:57:831.129 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for security-group 2024-10-21 Mon 02:02:57:831.134 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-group : security-group-access-control-list -> self 2024-10-21 Mon 02:02:57:831.139 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap access-control-list : self -> security-group-access-control-list 2024-10-21 Mon 02:02:57:831.144 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap access-control-list : security-group-access-control-list -> nil 2024-10-21 Mon 02:02:57:831.151 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for tag 2024-10-21 Mon 02:02:57:831.156 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : application-policy-set-tag -> self 2024-10-21 Mon 02:02:57:831.162 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:57:831.168 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : policy-management-application-policy-set -> application-policy-set-tag 2024-10-21 Mon 02:02:57:831.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap policy-management : policy-management-application-policy-set -> nil 2024-10-21 Mon 02:02:57:831.179 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.185 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2024-10-21 Mon 02:02:57:831.190 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:831.196 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:831.201 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:57:831.206 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:57:831.212 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.217 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2024-10-21 Mon 02:02:57:831.223 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:57:831.231 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine : virtual-machine-interface-virtual-machine -> nil 2024-10-21 Mon 02:02:57:831.236 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.241 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2024-10-21 Mon 02:02:57:831.246 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2024-10-21 Mon 02:02:57:831.251 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> nil 2024-10-21 Mon 02:02:57:831.257 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.262 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2024-10-21 Mon 02:02:57:831.267 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:831.272 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:831.277 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2024-10-21 Mon 02:02:57:831.283 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2024-10-21 Mon 02:02:57:831.289 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.293 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-routing-instance -> self 2024-10-21 Mon 02:02:57:831.300 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface-routing-instance : self -> virtual-machine-interface-routing-instance 2024-10-21 Mon 02:02:57:831.305 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface-routing-instance : virtual-machine-interface-routing-instance -> virtual-machine-interface-routing-instance 2024-10-21 Mon 02:02:57:831.310 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : self -> virtual-machine-interface-routing-instance 2024-10-21 Mon 02:02:57:831.316 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : virtual-machine-interface-routing-instance -> nil 2024-10-21 Mon 02:02:57:831.322 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.326 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-group -> self 2024-10-21 Mon 02:02:57:831.332 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-group : self -> virtual-machine-interface-security-group 2024-10-21 Mon 02:02:57:831.339 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-group : virtual-machine-interface-security-group -> nil 2024-10-21 Mon 02:02:57:831.346 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.350 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : alias-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.356 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap alias-ip : self -> alias-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.362 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap alias-ip : alias-ip-pool-alias-ip -> alias-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.368 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap alias-ip-pool : virtual-network-alias-ip-pool -> alias-ip-pool-alias-ip 2024-10-21 Mon 02:02:57:831.373 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-alias-ip-pool 2024-10-21 Mon 02:02:57:831.379 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-alias-ip-pool -> nil 2024-10-21 Mon 02:02:57:831.386 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.391 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.399 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.406 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.415 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2024-10-21 Mon 02:02:57:831.423 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:57:831.431 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:57:831.438 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2024-10-21 Mon 02:02:57:831.446 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2024-10-21 Mon 02:02:57:831.452 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.457 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.463 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.471 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.477 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2024-10-21 Mon 02:02:57:831.482 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:57:831.487 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-floating-ip-pool 2024-10-21 Mon 02:02:57:831.492 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:57:831.497 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2024-10-21 Mon 02:02:57:831.504 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.509 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.514 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.519 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap floating-ip : instance-ip-floating-ip -> floating-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.524 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap instance-ip : instance-ip-virtual-network -> instance-ip-floating-ip 2024-10-21 Mon 02:02:57:831.530 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> instance-ip-virtual-network 2024-10-21 Mon 02:02:57:831.535 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : instance-ip-virtual-network -> nil 2024-10-21 Mon 02:02:57:831.540 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.545 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : instance-ip-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.551 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap instance-ip : self -> instance-ip-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.556 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap instance-ip : instance-ip-virtual-machine-interface -> nil 2024-10-21 Mon 02:02:57:831.562 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.572 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-route-table -> self 2024-10-21 Mon 02:02:57:831.578 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap interface-route-table : self -> virtual-machine-interface-route-table 2024-10-21 Mon 02:02:57:831.583 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap interface-route-table : virtual-machine-interface-route-table -> nil 2024-10-21 Mon 02:02:57:831.594 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.599 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : subnet-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.605 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap subnet : self -> subnet-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.610 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap subnet : subnet-virtual-machine-interface -> nil 2024-10-21 Mon 02:02:57:831.617 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.622 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.628 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> logical-interface-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.634 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2024-10-21 Mon 02:02:57:831.640 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:57:831.645 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:57:831.651 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.666 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-port-group-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.672 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> virtual-port-group-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.677 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : physical-router-physical-interface -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:57:831.683 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:57:831.688 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:57:831.694 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.699 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.705 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.710 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bgp-as-a-service : bgpaas-bgp-router -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.719 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bgp-router : self -> bgpaas-bgp-router 2024-10-21 Mon 02:02:57:831.725 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bgp-router : instance-bgp-router -> bgpaas-bgp-router 2024-10-21 Mon 02:02:57:831.730 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : self -> instance-bgp-router 2024-10-21 Mon 02:02:57:831.735 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : instance-bgp-router -> nil 2024-10-21 Mon 02:02:57:831.741 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.746 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:831.751 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.756 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bgp-as-a-service : bgpaas-health-check -> bgpaas-virtual-machine-interface 2024-10-21 Mon 02:02:57:831.766 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap service-health-check : self -> bgpaas-health-check 2024-10-21 Mon 02:02:57:831.771 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap service-health-check : bgpaas-health-check -> nil 2024-10-21 Mon 02:02:57:831.778 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.782 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-qos-config -> self 2024-10-21 Mon 02:02:57:831.788 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap qos-config : self -> virtual-machine-interface-qos-config 2024-10-21 Mon 02:02:57:831.793 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap qos-config : virtual-machine-interface-qos-config -> nil 2024-10-21 Mon 02:02:57:831.799 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.804 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-logging-object -> self 2024-10-21 Mon 02:02:57:831.809 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : self -> virtual-machine-interface-security-logging-object 2024-10-21 Mon 02:02:57:831.814 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : virtual-machine-interface-security-logging-object -> nil 2024-10-21 Mon 02:02:57:831.821 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.826 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2024-10-21 Mon 02:02:57:831.832 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:57:831.840 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:57:831.846 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:57:831.851 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:57:831.856 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:57:831.862 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2024-10-21 Mon 02:02:57:831.867 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.872 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2024-10-21 Mon 02:02:57:831.877 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:57:831.882 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:57:831.887 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2024-10-21 Mon 02:02:57:831.893 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bridge-domain : virtual-machine-interface-bridge-domain -> nil 2024-10-21 Mon 02:02:57:831.899 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.904 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-tag -> self 2024-10-21 Mon 02:02:57:831.910 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : self -> virtual-machine-interface-tag 2024-10-21 Mon 02:02:57:831.915 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-interface-tag 2024-10-21 Mon 02:02:57:831.920 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:57:831.925 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:57:831.939 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:831.944 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:831.953 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:831.958 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:57:831.965 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:831.970 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2024-10-21 Mon 02:02:57:831.975 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:831.981 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-tag -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:831.986 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : self -> virtual-network-tag 2024-10-21 Mon 02:02:57:831.991 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : application-policy-set-tag -> virtual-network-tag 2024-10-21 Mon 02:02:57:831.996 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.001 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.007 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.011 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.017 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.022 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:57:832.028 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:832.033 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2024-10-21 Mon 02:02:57:832.038 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:57:832.044 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:57:832.049 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : self -> virtual-machine-tag 2024-10-21 Mon 02:02:57:832.054 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2024-10-21 Mon 02:02:57:832.062 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.067 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.072 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.077 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.082 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.087 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:57:832.094 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:832.099 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2024-10-21 Mon 02:02:57:832.104 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2024-10-21 Mon 02:02:57:832.109 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> virtual-machine-interface-sub-interface 2024-10-21 Mon 02:02:57:832.114 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:57:832.119 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2024-10-21 Mon 02:02:57:832.124 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : self -> virtual-machine-tag 2024-10-21 Mon 02:02:57:832.129 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2024-10-21 Mon 02:02:57:832.134 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.139 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.144 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.149 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.154 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.159 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:57:832.169 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:832.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : project-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:832.180 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap project : self -> project-virtual-machine-interface 2024-10-21 Mon 02:02:57:832.185 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap project : project-tag -> project-virtual-machine-interface 2024-10-21 Mon 02:02:57:832.190 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : self -> project-tag 2024-10-21 Mon 02:02:57:832.195 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : application-policy-set-tag -> project-tag 2024-10-21 Mon 02:02:57:832.200 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.205 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2024-10-21 Mon 02:02:57:832.210 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.215 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.220 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:832.225 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:57:832.231 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:832.236 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> self 2024-10-21 Mon 02:02:57:832.242 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2024-10-21 Mon 02:02:57:832.247 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2024-10-21 Mon 02:02:57:832.253 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2024-10-21 Mon 02:02:57:832.258 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> nil 2024-10-21 Mon 02:02:57:832.264 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for virtual-machine-interface 2024-10-21 Mon 02:02:57:832.269 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : logical-router-interface -> self 2024-10-21 Mon 02:02:57:832.275 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-router : self -> logical-router-interface 2024-10-21 Mon 02:02:57:832.283 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-router : logical-router-interface -> nil 2024-10-21 Mon 02:02:57:832.289 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for physical-interface 2024-10-21 Mon 02:02:57:832.294 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : physical-router-physical-interface -> self 2024-10-21 Mon 02:02:57:832.299 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:57:832.304 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:57:832.310 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for physical-interface 2024-10-21 Mon 02:02:57:832.315 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : virtual-port-group-physical-interface -> self 2024-10-21 Mon 02:02:57:832.321 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-port-group-physical-interface : self -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:57:832.326 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-port-group-physical-interface : virtual-port-group-physical-interface -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:57:832.331 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-port-group : self -> virtual-port-group-physical-interface 2024-10-21 Mon 02:02:57:832.336 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> nil 2024-10-21 Mon 02:02:57:832.342 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for logical-interface 2024-10-21 Mon 02:02:57:832.347 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> self 2024-10-21 Mon 02:02:57:832.352 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : self -> physical-interface-logical-interface 2024-10-21 Mon 02:02:57:832.357 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2024-10-21 Mon 02:02:57:832.362 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:57:832.367 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:57:832.373 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for logical-interface 2024-10-21 Mon 02:02:57:832.405 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-interface : logical-interface-virtual-machine-interface -> self 2024-10-21 Mon 02:02:57:832.411 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : self -> logical-interface-virtual-machine-interface 2024-10-21 Mon 02:02:57:832.420 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> nil 2024-10-21 Mon 02:02:57:832.426 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for logical-interface 2024-10-21 Mon 02:02:57:832.431 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap logical-interface : physical-router-logical-interface -> self 2024-10-21 Mon 02:02:57:832.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : self -> physical-router-logical-interface 2024-10-21 Mon 02:02:57:832.442 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : physical-router-logical-interface -> nil 2024-10-21 Mon 02:02:57:832.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for physical-router 2024-10-21 Mon 02:02:57:832.452 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-router : physical-router-physical-interface -> self 2024-10-21 Mon 02:02:57:832.458 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : self -> physical-router-physical-interface 2024-10-21 Mon 02:02:57:832.463 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap physical-interface : physical-router-physical-interface -> nil 2024-10-21 Mon 02:02:57:832.469 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for service-health-check 2024-10-21 Mon 02:02:57:832.474 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap service-health-check : service-port-health-check -> self 2024-10-21 Mon 02:02:57:832.480 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2024-10-21 Mon 02:02:57:832.485 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> service-port-health-check 2024-10-21 Mon 02:02:57:832.491 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:832.496 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:57:832.501 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:57:832.507 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for service-health-check 2024-10-21 Mon 02:02:57:832.512 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap service-health-check : service-port-health-check -> self 2024-10-21 Mon 02:02:57:832.517 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2024-10-21 Mon 02:02:57:832.523 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> service-port-health-check 2024-10-21 Mon 02:02:57:832.528 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2024-10-21 Mon 02:02:57:832.536 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2024-10-21 Mon 02:02:57:832.541 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2024-10-21 Mon 02:02:57:832.546 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> port-tuple-interface 2024-10-21 Mon 02:02:57:832.551 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2024-10-21 Mon 02:02:57:832.556 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2024-10-21 Mon 02:02:57:832.561 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2024-10-21 Mon 02:02:57:832.568 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for qos-config 2024-10-21 Mon 02:02:57:832.572 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap qos-config : global-qos-config-qos-config -> self 2024-10-21 Mon 02:02:57:832.579 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap global-qos-config : global-qos-config-qos-config -> nil 2024-10-21 Mon 02:02:57:832.587 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for forwarding-class 2024-10-21 Mon 02:02:57:832.592 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap forwarding-class : forwarding-class-qos-queue -> self 2024-10-21 Mon 02:02:57:832.598 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap qos-queue : self -> forwarding-class-qos-queue 2024-10-21 Mon 02:02:57:832.631 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap qos-queue : forwarding-class-qos-queue -> nil 2024-10-21 Mon 02:02:57:832.643 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:57:832.649 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : security-logging-object-network-policy -> self 2024-10-21 Mon 02:02:57:832.662 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap network-policy : self -> security-logging-object-network-policy 2024-10-21 Mon 02:02:57:832.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap network-policy : security-logging-object-network-policy -> nil 2024-10-21 Mon 02:02:57:832.674 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:57:832.679 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : security-logging-object-security-group -> self 2024-10-21 Mon 02:02:57:832.685 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-group : self -> security-logging-object-security-group 2024-10-21 Mon 02:02:57:832.691 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-group : security-logging-object-security-group -> nil 2024-10-21 Mon 02:02:57:832.701 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:57:832.706 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : firewall-policy-security-logging-object -> self 2024-10-21 Mon 02:02:57:832.713 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-security-logging-object : self -> firewall-policy-security-logging-object 2024-10-21 Mon 02:02:57:832.718 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-security-logging-object : firewall-policy-security-logging-object -> firewall-policy-security-logging-object 2024-10-21 Mon 02:02:57:832.724 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : firewall-policy-security-logging-object -> nil 2024-10-21 Mon 02:02:57:832.730 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for security-logging-object 2024-10-21 Mon 02:02:57:832.735 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap security-logging-object : firewall-rule-security-logging-object -> self 2024-10-21 Mon 02:02:57:832.741 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule-security-logging-object : self -> firewall-rule-security-logging-object 2024-10-21 Mon 02:02:57:832.897 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule-security-logging-object : firewall-rule-security-logging-object -> firewall-rule-security-logging-object 2024-10-21 Mon 02:02:57:832.907 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : firewall-rule-security-logging-object -> nil 2024-10-21 Mon 02:02:57:832.927 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for bridge-domain 2024-10-21 Mon 02:02:57:832.943 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2024-10-21 Mon 02:02:57:832.949 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:57:832.955 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:57:833.003 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2024-10-21 Mon 02:02:57:833.010 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2024-10-21 Mon 02:02:57:833.016 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for bridge-domain 2024-10-21 Mon 02:02:57:833.021 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2024-10-21 Mon 02:02:57:833.027 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2024-10-21 Mon 02:02:57:833.032 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2024-10-21 Mon 02:02:57:833.039 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for application-policy-set 2024-10-21 Mon 02:02:57:833.050 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> self 2024-10-21 Mon 02:02:57:833.056 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:833.061 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:833.066 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2024-10-21 Mon 02:02:57:833.071 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2024-10-21 Mon 02:02:57:833.079 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:57:833.084 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:57:833.091 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.096 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.101 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.106 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : firewall-rule-service-group -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.113 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap service-group : self -> firewall-rule-service-group 2024-10-21 Mon 02:02:57:833.118 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap service-group : firewall-rule-service-group -> nil 2024-10-21 Mon 02:02:57:833.125 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:57:833.130 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:57:833.135 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.140 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.145 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.151 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : firewall-rule-address-group -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.157 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap address-group : self -> firewall-rule-address-group 2024-10-21 Mon 02:02:57:833.166 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap address-group : address-group-tag -> firewall-rule-address-group 2024-10-21 Mon 02:02:57:833.171 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : self -> address-group-tag 2024-10-21 Mon 02:02:57:833.176 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : address-group-tag -> nil 2024-10-21 Mon 02:02:57:833.182 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:57:833.187 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:57:833.192 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.197 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.202 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.207 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : firewall-rule-tag -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.213 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : self -> firewall-rule-tag 2024-10-21 Mon 02:02:57:833.218 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap tag : firewall-rule-tag -> nil 2024-10-21 Mon 02:02:57:833.224 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Updating dependency tacker rules for firewall-policy 2024-10-21 Mon 02:02:57:833.228 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2024-10-21 Mon 02:02:57:833.234 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.238 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.244 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2024-10-21 Mon 02:02:57:833.249 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Adding ReactorMap firewall-rule : firewall-policy-firewall-rule -> nil 2024-10-21 Mon 02:02:57:984.697 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 10.0.0.30:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2024-10-21 Mon 02:02:57:984.919 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Current receive sock buffer size is 262144 2024-10-21 Mon 02:02:57:985.036 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Vrouter family is 34 2024-10-21 Mon 02:02:57:985.070 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575652075264, Pid 1095487]: KsyncTxQueue CPU pinning policy <>. KsyncTxQueuen not pinned to CPU 2024-10-21 Mon 02:02:57:985.110 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Flow table size : 20185088 2024-10-21 Mon 02:02:58:001.253 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Flow table size : 161218560 2024-10-21 Mon 02:02:58:001.307 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SetHugePages: Bridge table size:20185088 Flow table size:161218560 2024-10-21 Mon 02:02:58:001.326 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Mem mapping hugepage file:/dev/hugepages2M/bridge size:20185088 2024-10-21 Mon 02:02:58:001.364 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Mem mapped hugepage file:/dev/hugepages2M/bridge to addr:0x7fda4f000000 2024-10-21 Mon 02:02:58:001.374 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Mem mapping hugepage file:/dev/hugepages2M/flow size:161218560 2024-10-21 Mon 02:02:58:001.382 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Mem mapped hugepage file:/dev/hugepages2M/flow to addr:0x7fda45600000 2024-10-21 Mon 02:02:58:001.395 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Sending Huge Page configuration to VROUTER 2024-10-21 Mon 02:02:58:001.441 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Vrouter using already allocated memory 2024-10-21 Mon 02:02:58:017.900 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Flow table size : 161218560 2024-10-21 Mon 02:02:58:018.122 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Flow table size : 20185088 2024-10-21 Mon 02:02:58:082.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: ROLE : Generator 2024-10-21 Mon 02:02:58:082.055 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: MODULE : contrail-vrouter-agent 2024-10-21 Mon 02:02:58:082.061 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: SOURCE : cn-jenkins-deploy-platform-juju-os-439-1 2024-10-21 Mon 02:02:58:082.066 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: NODE TYPE : Compute 2024-10-21 Mon 02:02:58:082.071 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: INSTANCE ID : 0 2024-10-21 Mon 02:02:58:082.077 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: HTTP SERVER PORT : 8085 2024-10-21 Mon 02:02:58:082.083 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: System Log Send Rate Limit: 0 -> 0 2024-10-21 Mon 02:02:58:082.483 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: HTTP Introspect Init 2024-10-21 Mon 02:02:58:082.676 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:8085 Initialization complete src/contrail-common/io/tcp_server.cc 137 2024-10-21 Mon 02:02:58:082.730 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Sandesh Http Server Port: 8085 2024-10-21 Mon 02:02:58:082.814 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: NOT Writing http_port 8085TO : /tmp/contrail-vrouter-agent.1095055.http_port 2024-10-21 Mon 02:02:58:082.822 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: CONNECT TO COLLECTOR: 1 2024-10-21 Mon 02:02:58:082.908 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Idle 2024-10-21 Mon 02:02:58:083.107 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: No Client: 1729476178083032 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:58:083.491 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvStart in state Idle 2024-10-21 Mon 02:02:58:083.584 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Disconnect 2024-10-21 Mon 02:02:58:083.670 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvCollectorUpdate in state Disconnect 2024-10-21 Mon 02:02:58:083.918 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:02:58:084.013 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:084.048 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178083500 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:__default__ index = 0 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2024-10-21 Mon 02:02:58:084.059 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2024-10-21 Mon 02:02:58:084.074 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:084.092 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178083537 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:ip-fabric index = 1 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2024-10-21 Mon 02:02:58:084.099 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2024-10-21 Mon 02:02:58:084.107 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:084.136 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178083573 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2024-10-21 Mon 02:02:58:084.144 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:084.153 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:084.267 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178083661 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:58:084.282 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:58:084.297 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:084.428 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178083708 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:58:084.454 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:58:084.467 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:084.433 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface lo flags 65609 2024-10-21 Mon 02:02:58:084.521 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface ens3 flags 69699 2024-10-21 Mon 02:02:58:084.494 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178083730 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2024-10-21 Mon 02:02:58:084.585 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface fan-252 flags 69699 2024-10-21 Mon 02:02:58:084.616 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:084.623 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface ftun0 flags 69699 2024-10-21 Mon 02:02:58:084.647 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:084.689 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface lxdbr0 flags 4099 2024-10-21 Mon 02:02:58:084.699 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vethceed3bb2 flags 69699 2024-10-21 Mon 02:02:58:084.740 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vethff94287d flags 69699 2024-10-21 Mon 02:02:58:084.748 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface veth567cfff1 flags 69699 2024-10-21 Mon 02:02:58:084.763 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178084001 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:58:084.784 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:58:084.788 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vethbac30548 flags 69699 2024-10-21 Mon 02:02:58:084.813 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface veth402290ab flags 69699 2024-10-21 Mon 02:02:58:084.843 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface veth4cf19a6f flags 69699 2024-10-21 Mon 02:02:58:084.849 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vethc2ae4048 flags 69699 2024-10-21 Mon 02:02:58:084.879 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vethc1b275bc flags 69699 2024-10-21 Mon 02:02:58:084.885 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vethe9446158 flags 69699 2024-10-21 Mon 02:02:58:084.911 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface veth712d04e2 flags 69699 2024-10-21 Mon 02:02:58:084.917 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vethd92a2e9b flags 69699 2024-10-21 Mon 02:02:58:084.950 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface veth64775acc flags 69699 2024-10-21 Mon 02:02:58:084.956 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface docker0 flags 4099 2024-10-21 Mon 02:02:58:085.014 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface pkt1 flags 65601 2024-10-21 Mon 02:02:58:085.020 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface pkt3 flags 65601 2024-10-21 Mon 02:02:58:085.025 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface pkt2 flags 65601 2024-10-21 Mon 02:02:58:085.033 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface vhost0 flags 69699 2024-10-21 Mon 02:02:58:085.038 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink interface message RTM_NEWLINK for interface pkt0 flags 69699 2024-10-21 Mon 02:02:58:085.082 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface lo 2024-10-21 Mon 02:02:58:085.101 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface fan-252 2024-10-21 Mon 02:02:58:085.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface lxdbr0 2024-10-21 Mon 02:02:58:085.123 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface docker0 2024-10-21 Mon 02:02:58:085.132 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface vhost0 2024-10-21 Mon 02:02:58:085.145 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface lo 2024-10-21 Mon 02:02:58:085.154 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface fan-252 2024-10-21 Mon 02:02:58:085.163 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface ftun0 2024-10-21 Mon 02:02:58:085.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface vhost0 2024-10-21 Mon 02:02:58:085.183 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink address message RTM_NEWADDR for interface pkt0 2024-10-21 Mon 02:02:58:085.229 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2024-10-21 Mon 02:02:58:085.243 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.255 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.267 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 10.0.0.1/32 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.278 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 10.0.0.2/32 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.288 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 10.0.0.4/32 Interface vhost0 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.299 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 10.25.16.0/24 Interface lxdbr0 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.305 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2024-10-21 Mon 02:02:58:085.316 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 172.17.0.0/16 Interface docker0 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.326 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Handle netlink route message RTM_NEWROUTE : 252.0.0.0/8 Interface fan-252 GW 0.0.0.0 2024-10-21 Mon 02:02:58:085.332 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:58:085.337 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:58:085.346 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:58:085.351 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:58:085.356 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:58:085.361 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:58:085.366 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:58:085.371 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:58:085.376 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:58:085.380 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2024-10-21 Mon 02:02:58:085.385 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2024-10-21 Mon 02:02:58:085.396 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:58:085.401 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:58:085.406 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:58:085.411 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:58:085.416 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2024-10-21 Mon 02:02:58:085.420 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2024-10-21 Mon 02:02:58:085.425 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2024-10-21 Mon 02:02:58:085.430 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2024-10-21 Mon 02:02:58:085.435 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2024-10-21 Mon 02:02:58:085.440 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2024-10-21 Mon 02:02:58:085.445 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2024-10-21 Mon 02:02:58:085.450 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2024-10-21 Mon 02:02:58:085.455 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:58:085.460 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:58:085.468 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:58:085.473 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:58:085.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2024-10-21 Mon 02:02:58:106.282 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:9091 Initialization complete src/contrail-common/io/tcp_server.cc 137 2024-10-21 Mon 02:02:58:106.565 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Router ID configured. Connection to controller initiated 2024-10-21 Mon 02:02:58:108.063 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:108.108 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178107956 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:108.121 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:108.139 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:108.166 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178108036 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.30, Collector connection down, No Configuration for self Number of connections:2, Expected: 3 ], ] ] ] 2024-10-21 Mon 02:02:58:108.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:108.323 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:108.344 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178108277 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:108.352 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:108.362 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:108.382 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178108310 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.30, XMPP:dns-server:10.0.0.30, Collector connection down, No Configuration for self ], ] ] ] 2024-10-21 Mon 02:02:58:108.399 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:109.452 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server [fe80::f816:3eff:fec4:d814%vhost0]:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2024-10-21 Mon 02:02:58:255.019 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: VRouter [SYS_INFO]: VersionInfoLog: {"build-info":[{"build-version":"nightly","build-time":"2024-10-21 00:13:21.399567","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-6068-1.novalocal","build-id":"nightly-nightly.el7","build-number":"@contrail"}]} src/contrail-common/base/misc_utils.cc 41 2024-10-21 Mon 02:02:58:255.488 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:255.533 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178255464 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:255.546 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:255.719 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:255.746 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178255707 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect event = Connect Event local_port = 44925 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:255.757 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:255.771 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:255.791 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178255733 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:255.801 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:256.063 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:256.723 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476178255971 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect event = Connect Event local_port = 34383 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:256.750 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:261.969 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:44925::10.0.0.30:5269 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2024-10-21 Mon 02:02:58:262.017 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.30 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-10-21 Mon 02:02:58:262.199 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:262.241 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476178262168 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 44925 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:262.251 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:265.809 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:34383::10.0.0.30:8093 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2024-10-21 Mon 02:02:58:265.858 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.30 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-10-21 Mon 02:02:58:266.133 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:266.170 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178266101 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 34383 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:266.180 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:267.437 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:267.490 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476178267402 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:267.505 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:267.519 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:267.538 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476178267435 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:267.559 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:267.453 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1/dns:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:58:267.698 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:267.722 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476178267639 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:267.733 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:267.775 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:58:267.799 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:267.895 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1/dns:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:58:268.381 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178267758 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:268.410 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:268.427 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2024-10-21 Mon 02:02:58:268.428 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:268.489 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178267870 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-dns@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:268.499 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:268.514 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:268.534 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178268407 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-control@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:268.540 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:274.200 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV success controller/src/xmpp/xmpp_connection.cc 402 2024-10-21 Mon 02:02:58:274.439 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:274.522 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476178274400 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:274.535 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:275.095 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:275.129 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178275059 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:275.139 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:275.147 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:275.173 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178275108 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.30, Collector connection down, No Configuration for self ], ] ] ] 2024-10-21 Mon 02:02:58:275.180 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:276.557 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-439-1:10.0.0.30 RECV success controller/src/xmpp/xmpp_connection.cc 402 2024-10-21 Mon 02:02:58:276.679 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:276.735 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476178276655 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:276.748 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:277.631 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:277.665 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476178277617 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.30 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-439-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-10-21 Mon 02:02:58:277.674 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2024-10-21 Mon 02:02:58:277.716 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:277.792 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476178277684 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2024-10-21 Mon 02:02:58:277.812 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:325.575 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:325.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476178325543 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:02:58:325.641 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:370.670 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:370.765 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476178370520 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:02:58:370.781 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:58:371.712 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:371.814 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476178370874 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Addition uuid = 1e338ece-fda8-4842-aa92-7ce58d0af31f name = default-domain:default-project:ip-fabric acl_uuid = 561b7c53-af63-41b3-9c25-ec39072a7243 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2024-10-21 Mon 02:02:58:372.240 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2024-10-21 Mon 02:02:58:382.479 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:02:58:382.652 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476178381982 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 1e338ece-fda8-4842-aa92-7ce58d0af31f name = default-domain:default-project:ip-fabric acl_uuid = 561b7c53-af63-41b3-9c25-ec39072a7243 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2024-10-21 Mon 02:02:58:382.672 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2024-10-21 Mon 02:02:59:084.347 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:02:59:084.445 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:02:59:084.459 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:02:59:084.718 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Idle 2024-10-21 Mon 02:02:59:084.948 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:02:59:085.073 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476179084504 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:02:59:085.101 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:02:59:085.115 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:02:59:085.129 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:02:59:085.498 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476179084876 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 33, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:02:59:085.529 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:02:59:085.586 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2024-10-21 Mon 02:03:00:084.843 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:03:00:085.167 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:03:00:085.279 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:00:085.316 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476180084922 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:00:085.326 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:00:085.351 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:00:085.524 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476180085267 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 35, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:00:085.545 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:02:085.304 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:02:085.406 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:02:085.460 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:03:02:085.657 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Idle 2024-10-21 Mon 02:03:02:085.679 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2024-10-21 Mon 02:03:02:085.828 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:02:085.908 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476182085522 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:02:085.934 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:02:085.948 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:03:02:086.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:02:086.329 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476182085808 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 37, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:02:086.358 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:04:085.777 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:03:04:086.297 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:03:04:086.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:04:086.433 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476184085852 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:04:086.448 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:04:086.463 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:04:086.649 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476184086387 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 39, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:04:086.670 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:08:086.443 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:08:086.540 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:08:086.576 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:03:08:086.698 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Idle 2024-10-21 Mon 02:03:08:086.718 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2024-10-21 Mon 02:03:08:086.824 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:08:086.853 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476188086627 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:08:086.862 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:08:086.871 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:03:08:086.908 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:08:087.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476188086814 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 17, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 41, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:08:087.145 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:11:051.373 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:40502 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:03:11:094.117 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:40502 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:03:12:086.823 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:03:12:087.285 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:03:12:087.417 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:12:087.494 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476192086920 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:12:087.520 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:12:087.533 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:12:087.952 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476192087401 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 43, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:12:087.986 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:20:087.433 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:20:087.528 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:20:087.565 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:03:20:087.716 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Idle 2024-10-21 Mon 02:03:20:087.741 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2024-10-21 Mon 02:03:20:087.831 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:20:087.865 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476200087619 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:20:087.875 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:20:087.883 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:03:20:087.942 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:20:088.121 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476200087813 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 20, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 45, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:20:088.149 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:25:206.387 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:54060 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:03:25:249.851 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:54060 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:03:27:824.386 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:824.460 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476207824345 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric total_acl_rules = 4 interface_list= [ [ (*_iter66) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0, ] ] virtualmachine_list= [ [ ] ] acl = default-domain:default-project:ip-fabric:ip-fabric mirror_acl = ] 2024-10-21 Mon 02:03:27:824.475 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:03:27:824.491 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:824.534 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476207824417 [SYS_NOTICE]: UveVMInterfaceAgentTrace: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 ip_address = 10.0.0.30 virtual_network = default-domain:default-project:ip-fabric mac_address = fa:16:3e:c4:d8:14 label = 17 active = 1 vm_name = l2_active = 1 uuid = 62338fc6-1c29-4a80-9c2c-c12c5608156e ip6_address = :: ip6_active = 1 floating_ips= [ [ ] ] vm_uuid = ip4_active = 1 is_health_check_active = 1 health_check_instance_list= [ [ ] ] admin_state = 1 fixed_ip4_list= [ [ (*_iter26) = 10.0.0.30, ] ] fixed_ip6_list= [ [ ] ] vn_uuid = 1e338ece-fda8-4842-aa92-7ce58d0af31f tx_vlan = 65535 rx_vlan = 65535 parent_interface = ens3 vhostuser_mode = 0 port_mirror_enabled = 0 ] 2024-10-21 Mon 02:03:27:824.561 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVMInterfaceAgentTrace 2024-10-21 Mon 02:03:27:824.572 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:824.585 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476207824449 [SYS_NOTICE]: UVEVMITags: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 app = tier = site = deployment = labels= [ [ ] ] custom_tags= [ [ ] ] ] 2024-10-21 Mon 02:03:27:824.592 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMITags 2024-10-21 Mon 02:03:27:825.549 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:825.601 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476207825508 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 connected_networks= [ [ (*_iter142) = default-domain:default-project:ip-fabric, ] ] vn_count = 1 ] 2024-10-21 Mon 02:03:27:825.611 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2024-10-21 Mon 02:03:27:825.631 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:825.647 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476207825568 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 interface_list= [ [ (*_iter143) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0, ] ] error_intf_list= [ [ ] ] no_config_intf_list= [ [ ] ] vmi_count= [ active = 1 ] down_interface_count = 0 ] 2024-10-21 Mon 02:03:27:825.654 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2024-10-21 Mon 02:03:27:825.998 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:826.100 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476207825959 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric associated_fip_count = 0 in_bandwidth_usage = 0 out_bandwidth_usage = 0 vrf_stats_list= [ [ [ name = default-domain:default-project:ip-fabric:ip-fabric diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:03:27:826.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:03:27:826.127 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:826.185 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476207825984 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = __UNKNOWN__ vrf_stats_list= [ [ [ name = __untitled__ diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ] 2024-10-21 Mon 02:03:27:826.195 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:03:27:826.206 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:826.335 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476207826102 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric port_bucket_bmap= [ udp_sport_bitmap= [ [ (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, ] ] udp_dport_bitmap= [ [ (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, ] ] tcp_sport_bitmap= [ [ (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, ] ] tcp_dport_bitmap= [ [ (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, ] ] ] raw_if_stats= [ in_pkts = 126 in_bytes = 9870 out_pkts = 121 out_bytes = 8950 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 0 out_bw_usage = 0 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:03:27:826.361 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:03:27:826.374 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:27:826.391 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476207826136 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:03:27:826.401 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:03:28:083.459 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:28:083.554 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476208083273 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 3 dequeues = 3 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 56 dequeues = 56 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 69 dequeues = 69 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476200087714 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476207826372 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:03:28:083.579 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:28:083.588 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:28:083.823 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476208083411 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 56, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:28:083.845 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:28:088.955 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:03:28:089.545 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:03:28:089.825 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:28:089.908 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476208089284 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:28:089.918 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:28:089.927 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:28:090.265 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476208089794 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 58, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:28:090.308 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:28:322.606 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:28:323.544 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476208322527 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 xmpp_peer_list= [ [ [ ip = 10.0.0.30 status = 1 setup_time = 1729476178277699 primary = 1 ], ] ] self_ip_list= [ [ (*_iter146) = 10.0.0.30, ] ] build_info = {"build-info":[{"build-version":"nightly","build-time":"2024-10-21 00:13:21.399567","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-6068-1.novalocal","build-id":"nightly-nightly.el7","build-number":"@contrail"}]} phy_if= [ [ [ name = ens3 mac_address = fa:16:3e:c4:d8:14 ], ] ] vhost_if= [ name = vhost0 mac_address = fa:16:3e:c4:d8:14 ] log_file = /var/log/contrail/vrouter-agent/contrail-vrouter-agent.log config_file = /etc/contrail/contrail-vrouter-agent.conf log_local = 1 log_level = SYS_DEBUG log_category = sandesh_http_port = 8085 dns_servers= [ [ (*_iter149) = 10.0.0.30, ] ] vhost_cfg= [ name = vhost0 ip = 10.0.0.30 ip_prefix_len = 24 gateway = 10.0.0.1 ] eth_name= [ [ (*_iter150) = ens3, ] ] tunnel_type = MPLSoGRE hypervisor = kvm control_ip = 10.0.0.30 dns_server_list_cfg= [ [ (*_iter152) = 10.0.0.30:53, ] ] control_node_list_cfg= [ [ (*_iter153) = 10.0.0.30:5269, ] ] hostname_cfg = cn-jenkins-deploy-platform-juju-os-439-1 flow_cache_timeout_cfg = 0 ll_max_system_flows_cfg = 16384 ll_max_vm_flows_cfg = 16384 max_vm_flows_cfg = 100 gateway_cfg_list= [ [ ] ] headless_mode_cfg = 1 collector_server_list_cfg= [ [ (*_iter155) = 10.0.0.30:8086, ] ] log_flow = 0 mode = VROUTER platform = HOST vr_limits= [ max_labels = 5120 max_nexthops = 524288 max_interfaces = 4355 max_vrfs = 4096 max_mirror_entries = 255 vrouter_max_bridge_entries = 262144 vrouter_max_oflow_bridge_entries = 53248 vrouter_build_info = {"build-info": [{"build-version": "nightly", "build-time": "2024-10-21 00:10:55.154948", "build-user": "root", "build-hostname": "jenkins-build-prepare-sandbox-6068-1.novalocal"}]} vrouter_max_flow_entries = 524288 vrouter_max_oflow_entries = 105472 vrouter_priority_tagging = 1 ] bgpaas_enabled = 0 port_mirror_enabled = 0 subcluster_name = vr_high_watermark = 80 vr_low_watermark = 75 vr_limit_exceeded_list= [ [ ] ] res_limit = 0 res_table_limit = 0 loopback_ip = 0.0.0.0 gateway_list= [ [ (*_iter161) = 10.0.0.1, ] ] ] 2024-10-21 Mon 02:03:28:323.587 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVrouterAgent 2024-10-21 Mon 02:03:28:323.602 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:28:323.763 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476208322624 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 126 in_bytes = 9870 out_tpkts = 121 out_bytes = 8950 exception_packets = 27 exception_packets_dropped = 10 exception_packets_allowed = 17 total_flows = 0 aged_flows = 0 uptime = 1729476177824253 phy_if_info= [ [ _iter254->first = ens3 [ speed = -1 duplexity = 255 ], ] ] raw_vhost_stats= [ name = vhost0 in_pkts = 126 in_bytes = 9870 out_pkts = 121 out_bytes = 8950 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 91 in_bytes = 7690 out_pkts = 128 out_bytes = 9865 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:03:28:323.822 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:03:28:323.834 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:03:28:323.871 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476208322719 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:03:28:323.879 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:03:44:089.928 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:44:089.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:03:44:090.011 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:03:44:090.241 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Idle 2024-10-21 Mon 02:03:44:090.367 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:44:090.402 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476224090176 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:03:44:090.411 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:03:44:090.420 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:03:44:090.470 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:44:090.962 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476224090353 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 63, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:44:090.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:57:826.106 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:57:826.190 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476237826047 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:03:57:826.207 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:03:57:826.228 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:57:826.335 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476237826217 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 146 in_bytes = 10918 out_pkts = 141 out_bytes = 9890 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 279 out_bw_usage = 250 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:03:57:826.352 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:03:57:826.365 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:57:826.399 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476237826227 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:03:57:826.409 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:03:58:083.753 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:58:083.858 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476238083560 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 68 dequeues = 68 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 84 dequeues = 84 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476224090237 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476237826363 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:03:58:083.873 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:58:083.881 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:58:084.211 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476238083703 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 68, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:03:58:084.241 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:03:58:323.635 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:58:323.739 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476238323564 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.73125 five_min_avg = 0.8575 fifteen_min_avg = 0.53875 ] cpu_share = 0.236162 ] ] 2024-10-21 Mon 02:03:58:323.764 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:03:58:323.775 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:58:323.794 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476238323626 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.236162 used_sys_mem = 4269432198 one_min_cpuload = 0.73125 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:03:58:323.804 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:03:58:323.814 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:58:323.976 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476238323664 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 146 in_bytes = 10918 out_tpkts = 141 out_bytes = 9890 exception_packets = 28 exception_packets_dropped = 11 raw_vhost_stats= [ name = vhost0 in_pkts = 146 in_bytes = 10918 out_pkts = 141 out_bytes = 9890 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 96 in_bytes = 8000 out_pkts = 148 out_bytes = 10913 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:03:58:323.993 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:03:58:324.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:03:58:324.035 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476238323704 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:03:58:324.043 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:04:00:090.426 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:04:00:090.740 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:04:00:090.893 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:00:090.928 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476240090581 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:04:00:090.937 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:04:00:090.946 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:00:091.232 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476240090864 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 74, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:04:00:091.253 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:04:27:826.318 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:27:826.385 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476267826263 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:04:27:826.397 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:04:27:826.469 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:27:826.616 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476267826443 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 169 in_bytes = 12160 out_pkts = 165 out_bytes = 11094 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 331 out_bw_usage = 321 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:04:27:826.644 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:04:27:826.655 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:27:826.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476267826448 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:04:27:826.675 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:04:28:084.128 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:28:084.310 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476268083938 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 79 dequeues = 79 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 96 dequeues = 96 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476240090590 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476267826653 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:04:28:084.328 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:04:28:084.337 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:28:084.686 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476268084077 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 79, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:04:28:084.734 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:04:28:323.880 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:28:324.099 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476268323823 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 169 in_bytes = 12160 out_tpkts = 165 out_bytes = 11094 exception_packets = 30 exception_packets_dropped = 12 exception_packets_allowed = 18 raw_vhost_stats= [ name = vhost0 in_pkts = 169 in_bytes = 12160 out_pkts = 165 out_bytes = 11094 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 6.34258e-16 total_out_bandwidth_utilization = 1.65341e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 117, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 305, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 105 in_bytes = 8574 out_pkts = 171 out_bytes = 12155 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:04:28:324.127 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:04:28:324.138 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:04:28:324.170 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476268323886 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:04:28:324.177 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:04:30:090.862 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:04:30:091.463 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:04:30:091.482 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:04:30:091.580 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Idle 2024-10-21 Mon 02:04:30:092.766 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:30:092.823 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476270091517 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:04:30:092.847 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:04:30:092.858 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:04:30:092.898 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:30:093.232 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476270091671 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 83, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:04:30:093.258 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:04:44:249.524 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:42738 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:04:44:297.928 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:42738 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:04:57:826.361 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:57:826.433 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476297826274 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:04:57:826.445 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:04:57:826.455 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:57:826.523 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476297826428 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 195 in_bytes = 13716 out_pkts = 191 out_bytes = 12494 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 429 out_bw_usage = 386 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:04:57:826.548 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:04:57:826.557 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:57:826.568 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476297826435 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:04:57:826.575 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:04:58:084.459 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:58:084.561 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476298084231 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 88 dequeues = 88 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 107 dequeues = 107 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476270091579 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476297826555 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:04:58:084.576 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:04:58:084.585 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:58:084.887 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476298084411 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 24, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 88, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:04:58:084.920 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:04:58:324.173 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:58:324.255 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476298324133 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.68 five_min_avg = 0.815 fifteen_min_avg = 0.54375 ] cpu_share = 0.204167 ] ] 2024-10-21 Mon 02:04:58:324.266 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:04:58:324.276 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:58:324.292 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476298324155 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.204167 used_sys_mem = 4269432198 one_min_cpuload = 0.68 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:04:58:324.300 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:04:58:324.308 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:58:324.455 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476298324190 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 195 in_bytes = 13716 out_tpkts = 191 out_bytes = 12494 exception_packets = 31 exception_packets_allowed = 19 raw_vhost_stats= [ name = vhost0 in_pkts = 195 in_bytes = 13716 out_pkts = 191 out_bytes = 12494 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 116 in_bytes = 9344 out_pkts = 197 out_bytes = 13711 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:04:58:324.477 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:04:58:324.487 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:04:58:324.518 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476298324244 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:04:58:324.528 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:04:59:506.029 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:40016 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:04:59:549.896 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:40016 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:05:00:091.720 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:05:00:092.060 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:05:00:092.244 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:00:092.294 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476300091824 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:05:00:092.307 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:05:00:092.321 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:00:092.776 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476300092223 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 26, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 94, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:05:00:092.868 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:05:13:598.050 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:38808 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:05:13:641.841 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:38808 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:05:27:826.464 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:27:826.530 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476327826401 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:05:27:826.543 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:05:27:826.552 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:27:826.624 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476327826527 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 227 in_bytes = 15692 out_pkts = 222 out_bytes = 14272 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 526 out_bw_usage = 474 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:05:27:826.637 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:05:27:826.647 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:27:826.659 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476327826530 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:05:27:826.676 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:05:28:084.916 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:28:085.091 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476328084692 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 99 dequeues = 99 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 119 dequeues = 119 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476300091837 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476327826646 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:05:28:085.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:05:28:085.129 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:28:085.579 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476328084837 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 28, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 99, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:05:28:085.615 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:05:28:324.397 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:28:324.597 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476328324330 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 227 in_bytes = 15692 out_tpkts = 222 out_bytes = 14272 exception_packets = 33 exception_packets_dropped = 13 exception_packets_allowed = 20 raw_vhost_stats= [ name = vhost0 in_pkts = 227 in_bytes = 15692 out_pkts = 222 out_bytes = 14272 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 1.38236e-15 total_out_bandwidth_utilization = 2.54788e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 255, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 470, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 132 in_bytes = 10492 out_pkts = 229 out_bytes = 15687 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:05:28:324.620 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:05:28:324.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:05:28:324.661 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476328324373 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:05:28:324.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:05:30:092.173 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:05:30:092.236 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:05:30:092.249 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:05:30:092.417 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Idle 2024-10-21 Mon 02:05:30:092.539 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:30:092.568 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476330092346 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:05:30:092.581 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:05:30:092.594 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:05:30:092.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:30:093.103 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476330092518 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 28, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 103, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:05:30:093.145 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:05:53:514.974 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:37916 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:05:53:561.967 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:37916 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:05:57:826.581 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:57:826.640 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476357826530 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:05:57:826.652 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:05:57:826.859 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:57:826.948 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476357826817 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 249 in_bytes = 16960 out_pkts = 244 out_bytes = 15432 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 338 out_bw_usage = 309 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:05:57:826.974 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:05:57:826.984 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:57:826.996 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476357826843 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:05:57:827.003 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:05:58:085.235 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:58:085.324 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476358085039 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 108 dequeues = 108 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 130 dequeues = 130 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476330092415 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476357826982 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:05:58:085.338 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:05:58:085.346 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:58:085.603 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476358085190 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 108, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:05:58:085.629 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:05:58:324.586 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:58:324.676 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476358324541 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.4575 five_min_avg = 0.72625 fifteen_min_avg = 0.53125 ] cpu_share = 0.197917 ] ] 2024-10-21 Mon 02:05:58:324.688 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:05:58:324.698 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:58:324.718 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476358324573 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.197917 used_sys_mem = 4269432198 one_min_cpuload = 0.4575 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:05:58:324.726 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:05:58:324.734 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:58:324.894 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476358324620 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 249 in_bytes = 16960 out_tpkts = 244 out_bytes = 15432 exception_packets = 34 exception_packets_allowed = 21 raw_vhost_stats= [ name = vhost0 in_pkts = 249 in_bytes = 16960 out_pkts = 244 out_bytes = 15432 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 139 in_bytes = 11022 out_pkts = 251 out_bytes = 16955 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:05:58:324.923 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:05:58:324.935 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:05:58:325.034 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476358324668 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:05:58:325.049 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:06:00:092.487 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:06:00:092.720 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:06:00:092.863 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:00:092.894 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476360092572 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:06:00:092.903 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:06:00:092.912 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:00:093.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476360092849 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 114, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:06:00:093.220 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:06:27:826.429 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:27:826.535 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476387826392 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:06:27:826.550 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:06:27:826.573 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:27:826.647 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476387826563 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 381 in_bytes = 28050 out_pkts = 376 out_bytes = 26414 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 3059 out_bw_usage = 3029 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:06:27:826.672 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:06:27:826.681 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:27:826.693 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476387826573 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:06:27:826.700 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:06:28:085.578 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:28:085.707 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476388085420 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 119 dequeues = 119 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 142 dequeues = 142 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476360092580 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476387826680 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:06:28:085.734 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:06:28:085.743 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:28:086.014 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476388085511 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 119, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:06:28:086.034 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:06:28:324.840 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:28:325.116 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476388324786 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 381 in_bytes = 28050 out_tpkts = 376 out_bytes = 26414 exception_packets = 35 exception_packets_allowed = 22 raw_vhost_stats= [ name = vhost0 in_pkts = 381 in_bytes = 28050 out_pkts = 376 out_bytes = 26414 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 7.86047e-15 total_out_bandwidth_utilization = 8.9284e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1450, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1647, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 256 in_bytes = 21374 out_pkts = 383 out_bytes = 28045 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:06:28:325.153 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:06:28:325.170 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:06:28:325.219 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476388324856 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:06:28:325.230 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:06:30:092.855 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:06:30:092.934 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:06:30:092.945 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:06:30:093.049 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Idle 2024-10-21 Mon 02:06:30:093.157 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:30:093.185 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476390092997 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:06:30:093.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:06:30:093.203 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:06:30:093.234 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:30:093.535 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476390093146 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 123, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:06:30:093.559 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:06:57:828.098 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:57:828.180 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476417828059 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:06:57:828.211 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:06:57:828.250 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:57:828.382 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476417828228 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 401 in_bytes = 29118 out_pkts = 396 out_bytes = 27374 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 284 out_bw_usage = 256 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:06:57:828.397 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:06:57:828.408 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:57:828.419 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476417828235 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:06:57:828.427 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:06:58:085.885 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:58:086.014 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476418085651 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 128 dequeues = 128 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 153 dequeues = 153 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476390093047 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476417828406 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:06:58:086.047 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:06:58:086.066 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:58:086.350 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476418085837 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 128, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:06:58:086.374 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:06:58:325.077 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:58:325.171 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476418325029 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.84375 five_min_avg = 0.83125 fifteen_min_avg = 0.58125 ] cpu_share = 0.199967 ] ] 2024-10-21 Mon 02:06:58:325.185 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:06:58:325.195 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:58:325.214 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476418325053 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.199967 used_sys_mem = 4269432198 one_min_cpuload = 0.84375 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:06:58:325.221 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:06:58:325.230 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:58:325.406 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476418325105 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 401 in_bytes = 29118 out_tpkts = 396 out_bytes = 27374 raw_vhost_stats= [ name = vhost0 in_pkts = 401 in_bytes = 29118 out_pkts = 396 out_bytes = 27374 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 261 in_bytes = 21704 out_pkts = 403 out_bytes = 29113 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:06:58:325.418 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:06:58:325.426 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:06:58:325.461 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476418325154 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:06:58:325.468 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:07:00:093.266 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:07:00:093.519 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:07:00:093.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:00:093.661 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476420093359 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:07:00:093.670 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:07:00:093.679 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:00:093.995 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476420093618 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 134, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:07:00:094.027 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:07:27:828.182 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:27:828.262 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476447828141 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:07:27:828.277 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:07:27:828.326 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:27:828.431 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476447828301 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 421 in_bytes = 30202 out_pkts = 415 out_bytes = 28316 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 289 out_bw_usage = 251 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:07:27:828.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:07:27:828.458 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:27:828.470 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476447828313 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:07:27:828.477 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:07:28:086.224 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:28:086.378 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476448086050 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 139 dequeues = 139 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 165 dequeues = 165 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476420093372 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476447828457 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:07:28:086.407 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:07:28:086.416 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:28:086.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476448086178 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 139, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:07:28:086.688 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:07:28:325.325 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:28:325.541 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476448325279 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 421 in_bytes = 30202 out_tpkts = 415 out_bytes = 28316 raw_vhost_stats= [ name = vhost0 in_pkts = 421 in_bytes = 30202 out_pkts = 415 out_bytes = 28316 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 4.60786e-16 total_out_bandwidth_utilization = 1.55041e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 85, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 286, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 265 in_bytes = 22016 out_pkts = 423 out_bytes = 30197 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:07:28:325.569 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:07:28:325.583 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:07:28:325.615 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476448325330 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:07:28:325.623 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:07:30:093.709 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:07:30:093.925 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:07:30:093.941 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:07:30:094.061 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Idle 2024-10-21 Mon 02:07:30:094.179 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:30:094.219 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476450093999 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:07:30:094.235 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:07:30:094.246 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:07:30:094.278 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:30:094.539 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476450094166 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 143, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:07:30:094.570 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:07:57:829.735 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:57:829.824 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476477829684 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:07:57:829.838 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:07:57:829.878 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:57:829.956 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476477829852 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 433 in_bytes = 30886 out_pkts = 427 out_bytes = 28892 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 182 out_bw_usage = 153 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:07:57:829.969 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:07:57:829.979 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:57:829.991 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476477829870 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:07:57:830.012 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:07:58:086.507 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:58:086.616 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476478086334 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 148 dequeues = 148 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 176 dequeues = 176 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476450094059 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476477829977 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:07:58:086.631 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:07:58:086.640 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:58:087.020 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476478086462 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 148, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:07:58:087.060 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:07:58:325.517 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:58:325.600 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476478325477 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.4175 five_min_avg = 0.71 fifteen_min_avg = 0.555 ] cpu_share = 0.202083 ] ] 2024-10-21 Mon 02:07:58:325.624 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:07:58:325.637 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:58:325.656 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476478325497 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.202083 used_sys_mem = 4269432198 one_min_cpuload = 0.4175 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:07:58:325.664 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:07:58:325.671 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:58:325.836 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476478325549 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 433 in_bytes = 30886 out_tpkts = 427 out_bytes = 28892 exception_packets = 37 exception_packets_dropped = 14 exception_packets_allowed = 23 raw_vhost_stats= [ name = vhost0 in_pkts = 433 in_bytes = 30886 out_pkts = 427 out_bytes = 28892 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 271 in_bytes = 22340 out_pkts = 435 out_bytes = 30881 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:07:58:325.852 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:07:58:325.860 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:07:58:325.891 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476478325592 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:07:58:325.898 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:08:00:094.214 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:08:00:094.468 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:08:00:094.578 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:00:094.613 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476480094305 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:08:00:094.632 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:08:00:094.642 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:00:094.926 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476480094565 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 154, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:08:00:094.951 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:08:27:831.364 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:27:831.440 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476507831327 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:08:27:831.452 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:08:27:831.566 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:27:831.685 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476507831540 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 440 in_bytes = 31360 out_pkts = 436 out_bytes = 29342 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 120 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:08:27:831.714 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:08:27:831.724 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:27:831.736 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476507831549 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:08:27:831.744 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:08:28:086.830 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:28:086.945 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476508086656 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 159 dequeues = 159 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 188 dequeues = 188 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476480094313 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476507831722 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:08:28:086.959 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:08:28:086.969 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:28:087.215 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476508086769 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 159, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:08:28:087.251 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:08:28:325.809 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:28:326.029 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476508325753 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 440 in_bytes = 31360 out_tpkts = 436 out_bytes = 29342 exception_packets = 39 exception_packets_allowed = 25 raw_vhost_stats= [ name = vhost0 in_pkts = 440 in_bytes = 31360 out_pkts = 436 out_bytes = 29342 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 399 out_bandwidth_usage = 575 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 4.66207e-16 total_out_bandwidth_utilization = 8.94467e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 86, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 165, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 277 in_bytes = 22664 out_pkts = 444 out_bytes = 31439 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:08:28:326.058 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:08:28:326.071 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:08:28:326.102 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476508325821 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:08:28:326.110 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:08:30:094.646 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:08:30:094.720 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:08:30:094.735 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:08:30:094.841 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Idle 2024-10-21 Mon 02:08:30:094.967 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:30:094.998 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476510094767 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:08:30:095.018 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:08:30:095.027 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:08:30:095.057 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:30:095.329 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476510094954 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 163, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:08:30:095.352 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:08:57:831.599 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:57:831.656 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476537831565 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:08:57:831.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:08:57:831.759 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:57:831.869 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476537831731 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 451 in_bytes = 32098 out_pkts = 449 out_bytes = 30112 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 196 out_bw_usage = 205 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:08:57:831.894 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:08:57:831.905 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:57:831.917 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476537831742 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:08:57:831.924 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:08:58:087.096 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:58:087.181 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476538086878 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 168 dequeues = 168 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 199 dequeues = 199 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476510094839 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476537831903 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:08:58:087.196 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:08:58:087.204 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:58:087.472 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476538087054 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 168, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:08:58:087.502 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:08:58:326.008 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:58:326.095 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476538325965 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.31125 five_min_avg = 0.6325 fifteen_min_avg = 0.53875 ] cpu_share = 0.191667 ] ] 2024-10-21 Mon 02:08:58:326.107 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:08:58:326.120 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:58:326.137 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476538325993 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.191667 used_sys_mem = 4269432198 one_min_cpuload = 0.31125 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:08:58:326.145 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:08:58:326.152 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:58:326.303 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476538326032 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 451 in_bytes = 32098 out_tpkts = 449 out_bytes = 30112 exception_packets = 40 exception_packets_allowed = 26 raw_vhost_stats= [ name = vhost0 in_pkts = 451 in_bytes = 32098 out_pkts = 449 out_bytes = 30112 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 287 in_bytes = 23308 out_pkts = 455 out_bytes = 32177 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:08:58:326.327 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:08:58:326.337 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:08:58:326.367 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476538326095 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:08:58:326.377 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:08:58:644.648 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:41288 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:08:58:689.851 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:41288 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:09:00:095.017 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:09:00:095.351 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:09:00:095.502 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:00:095.545 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476540095120 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:09:00:095.558 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:09:00:095.570 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:00:096.035 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476540095479 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 32, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 174, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:09:00:096.092 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:09:12:548.272 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:49138 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:09:12:593.893 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:49138 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:09:27:831.687 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:27:831.761 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476567831633 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:09:27:831.792 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:09:27:831.805 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:27:831.885 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476567831793 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 467 in_bytes = 33278 out_pkts = 467 out_bytes = 31324 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 314 out_bw_usage = 323 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:09:27:831.899 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:09:27:831.907 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:27:831.919 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476567831800 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:09:27:831.935 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:09:28:087.373 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:28:087.531 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476568087235 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 179 dequeues = 179 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 211 dequeues = 211 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476540095130 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476567831906 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:09:28:087.547 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:09:28:087.557 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:28:087.850 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476568087332 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 34, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 179, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:09:28:087.879 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:09:28:326.224 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:28:326.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476568326181 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 467 in_bytes = 33278 out_tpkts = 467 out_bytes = 31324 exception_packets = 41 exception_packets_allowed = 27 raw_vhost_stats= [ name = vhost0 in_pkts = 467 in_bytes = 33278 out_pkts = 467 out_bytes = 31324 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 1.24683e-15 total_out_bandwidth_utilization = 1.38236e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 230, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 255, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 302 in_bytes = 24394 out_pkts = 471 out_bytes = 33357 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:09:28:326.507 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:09:28:326.520 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:09:28:326.552 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476568326232 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:09:28:326.560 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:09:29:540.830 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:38684 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:09:29:585.944 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:38684 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:09:30:095.522 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:09:30:095.620 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:09:30:095.659 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:09:30:095.772 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Idle 2024-10-21 Mon 02:09:30:095.907 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:30:095.938 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476570095693 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:09:30:095.948 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:09:30:095.968 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:09:30:096.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:30:096.282 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476570095889 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 36, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 183, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:09:30:096.310 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:09:54:357.359 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:52380 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:09:54:405.919 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:52380 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:09:57:832.289 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:57:832.354 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476597832225 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:09:57:832.366 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:09:57:832.393 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:57:832.468 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476597832353 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 486 in_bytes = 34632 out_pkts = 486 out_bytes = 32570 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 361 out_bw_usage = 332 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:09:57:832.497 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:09:57:832.507 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:57:832.518 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476597832356 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:09:57:832.525 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:09:58:087.664 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:58:087.759 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476598087471 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 188 dequeues = 188 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 222 dequeues = 222 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476570095770 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476597832505 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:09:58:087.783 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:09:58:087.793 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:58:088.085 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476598087614 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 188, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:09:58:088.638 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:09:58:326.433 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:58:326.526 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476598326389 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.35125 five_min_avg = 0.58 fifteen_min_avg = 0.52625 ] cpu_share = 0.214583 ] ] 2024-10-21 Mon 02:09:58:326.539 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:09:58:326.549 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:58:326.567 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476598326414 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.214583 used_sys_mem = 4269432198 one_min_cpuload = 0.35125 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:09:58:326.576 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:09:58:326.583 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:58:326.757 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476598326451 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 486 in_bytes = 34632 out_tpkts = 486 out_bytes = 32570 exception_packets = 43 exception_packets_allowed = 29 raw_vhost_stats= [ name = vhost0 in_pkts = 486 in_bytes = 34632 out_pkts = 486 out_bytes = 32570 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 318 in_bytes = 25514 out_pkts = 490 out_bytes = 34711 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:09:58:326.790 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:09:58:326.799 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:09:58:326.830 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476598326510 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:09:58:326.841 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:10:00:095.951 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:10:00:096.256 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:10:00:096.394 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:00:096.427 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476600096061 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:10:00:096.436 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:10:00:096.446 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:00:096.732 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476600096380 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 194, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:10:00:096.761 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:10:27:832.216 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:27:832.380 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476627832120 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:10:27:832.411 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:10:27:832.466 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:27:832.576 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476627832292 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 494 in_bytes = 35148 out_pkts = 494 out_bytes = 32978 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 142 out_bw_usage = 112 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:10:27:832.594 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:10:27:832.609 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:27:832.626 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476627832297 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:10:27:832.636 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:10:28:088.082 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:28:088.274 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476628087863 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 199 dequeues = 199 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 234 dequeues = 234 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476600096070 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476627832607 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:10:28:088.319 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:10:28:088.336 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:28:088.852 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476628088024 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 199, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:10:28:088.883 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:10:28:326.694 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:28:326.949 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476628326646 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 494 in_bytes = 35148 out_tpkts = 494 out_bytes = 32978 raw_vhost_stats= [ name = vhost0 in_pkts = 494 in_bytes = 35148 out_pkts = 494 out_bytes = 32978 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 1.00831e-15 total_out_bandwidth_utilization = 1.34983e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 186, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 249, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 323 in_bytes = 25796 out_pkts = 498 out_bytes = 35227 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:10:28:326.982 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:10:28:326.993 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:10:28:327.026 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476628326705 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:10:28:327.037 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:10:30:096.484 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:10:30:096.618 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:10:30:096.683 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:10:30:096.885 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Idle 2024-10-21 Mon 02:10:30:097.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:30:097.236 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476630096750 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:10:30:097.255 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:10:30:097.274 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:10:30:097.322 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:30:097.845 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476630097140 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 203, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:10:30:097.908 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:10:57:833.893 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:57:833.976 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476657833852 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:10:57:833.987 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:10:57:834.050 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:57:834.156 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476657834025 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 504 in_bytes = 35844 out_pkts = 505 out_bytes = 33636 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 185 out_bw_usage = 175 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:10:57:834.171 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:10:57:834.182 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:57:834.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476657834031 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:10:57:834.201 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:10:58:089.432 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:58:089.512 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476658089062 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 208 dequeues = 208 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 245 dequeues = 245 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476630096878 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476657834180 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:10:58:089.539 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:10:58:089.548 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:58:089.846 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476658089384 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 56 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 208, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:10:58:089.868 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:10:58:327.098 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:58:327.230 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476658326866 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.29875 five_min_avg = 0.52875 fifteen_min_avg = 0.51125 ] cpu_share = 0.20625 ] ] 2024-10-21 Mon 02:10:58:327.248 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:10:58:327.265 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:58:327.293 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476658327050 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.20625 used_sys_mem = 4269432198 one_min_cpuload = 0.29875 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:10:58:327.326 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:10:58:327.351 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:58:327.646 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476658327096 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 504 in_bytes = 35844 out_tpkts = 505 out_bytes = 33636 exception_packets = 44 exception_packets_allowed = 30 raw_vhost_stats= [ name = vhost0 in_pkts = 504 in_bytes = 35844 out_pkts = 505 out_bytes = 33636 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 331 in_bytes = 26328 out_pkts = 508 out_bytes = 35923 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:10:58:327.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:10:58:327.682 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:10:58:327.733 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476658327134 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:10:58:327.743 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:11:00:097.061 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:11:00:097.405 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:11:00:097.570 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:00:097.631 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476660097173 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:11:00:097.646 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:11:00:097.658 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:00:098.106 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476660097548 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 214, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:11:00:098.149 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:11:27:834.062 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:27:834.121 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476687834006 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:11:27:834.132 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:11:27:834.141 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:27:834.212 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476687834122 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 511 in_bytes = 36318 out_pkts = 514 out_bytes = 34142 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 134 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:11:27:834.224 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:11:27:834.233 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:27:834.244 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476687834126 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:11:27:834.267 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:11:28:089.964 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:28:090.078 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476688089793 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 219 dequeues = 219 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 257 dequeues = 257 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476660097182 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476687834231 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:11:28:090.092 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:11:28:090.101 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:28:090.350 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476688089913 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 59 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 219, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:11:28:090.383 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:11:28:327.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:28:327.655 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476688327385 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 511 in_bytes = 36318 out_tpkts = 514 out_bytes = 34142 raw_vhost_stats= [ name = vhost0 in_pkts = 511 in_bytes = 36318 out_pkts = 514 out_bytes = 34142 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 6.55942e-16 total_out_bandwidth_utilization = 8.45678e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 121, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 156, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 337 in_bytes = 26708 out_pkts = 515 out_bytes = 36397 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:11:28:327.681 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:11:28:327.693 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:11:28:327.724 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476688327458 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:11:28:327.731 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:11:30:097.693 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:11:30:097.786 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:11:30:097.920 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:11:30:098.054 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Idle 2024-10-21 Mon 02:11:30:098.216 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:30:098.262 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476690097955 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:11:30:098.276 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:11:30:098.289 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:11:30:098.323 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:30:098.635 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476690098192 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 61 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 223, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:11:30:098.673 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:11:57:834.794 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:57:834.850 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476717834727 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:11:57:834.861 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:11:57:834.870 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:57:834.940 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476717834772 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 617 in_bytes = 45604 out_pkts = 621 out_bytes = 43426 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2476 out_bw_usage = 2475 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:11:57:834.963 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:11:57:834.973 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:57:834.984 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476717834775 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:11:57:834.991 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:11:58:090.219 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:58:090.305 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476718090047 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 228 dequeues = 228 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 268 dequeues = 268 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476690098051 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476717834971 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:11:58:090.318 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:11:58:090.327 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:58:090.589 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476718090172 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 62 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 228, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:11:58:090.632 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:11:58:327.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:58:327.890 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476718327564 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 1.12875 five_min_avg = 0.68875 fifteen_min_avg = 0.565 ] cpu_share = 0.204167 ] ] 2024-10-21 Mon 02:11:58:327.934 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:11:58:327.948 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:58:327.966 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476718327585 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.204167 used_sys_mem = 4269432198 one_min_cpuload = 1.12875 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:11:58:327.974 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:11:58:327.981 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:58:328.131 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476718327617 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 617 in_bytes = 45604 out_tpkts = 621 out_bytes = 43426 exception_packets = 46 exception_packets_allowed = 32 raw_vhost_stats= [ name = vhost0 in_pkts = 617 in_bytes = 45604 out_pkts = 621 out_bytes = 43426 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 441 in_bytes = 35866 out_pkts = 621 out_bytes = 45683 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:11:58:328.153 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:11:58:328.161 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:11:58:328.191 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476718327647 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:11:58:328.199 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:12:00:098.177 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:12:00:098.422 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:12:00:099.516 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:00:099.555 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476720098261 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:12:00:099.576 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:12:00:099.585 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:00:099.878 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476720099499 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 64 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 234, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:12:00:099.903 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:12:27:836.593 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:27:836.657 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476747836555 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:12:27:836.669 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:12:27:836.732 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:27:836.902 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476747836704 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 640 in_bytes = 47526 out_pkts = 644 out_bytes = 45240 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 512 out_bw_usage = 483 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:12:27:836.942 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:12:27:836.956 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:27:836.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476747836713 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:12:27:837.011 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:12:28:090.537 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:28:090.660 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476748090395 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 239 dequeues = 239 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 280 dequeues = 280 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476720098270 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476747836954 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:12:28:090.674 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:12:28:090.683 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:28:090.972 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476748090496 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 65 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 239, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:12:28:090.992 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:12:28:327.829 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:28:328.042 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476748327789 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 640 in_bytes = 47526 out_tpkts = 644 out_bytes = 45240 exception_packets = 47 exception_packets_dropped = 15 raw_vhost_stats= [ name = vhost0 in_pkts = 640 in_bytes = 47526 out_pkts = 644 out_bytes = 45240 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 7.83878e-15 total_out_bandwidth_utilization = 8.09899e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1446, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1494, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 461 in_bytes = 37554 out_pkts = 644 out_bytes = 47605 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:12:28:328.074 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:12:28:328.086 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:12:28:328.118 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476748327837 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:12:28:328.125 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:12:30:099.505 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:12:30:099.595 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:12:30:099.634 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:12:30:099.858 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Idle 2024-10-21 Mon 02:12:30:100.015 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:30:100.047 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476750099672 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:12:30:100.057 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:12:30:100.066 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:12:30:100.116 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:30:100.434 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476750099999 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 67 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 243, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:12:30:100.464 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:12:57:838.298 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:57:838.360 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476777838264 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:12:57:838.372 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:12:57:838.485 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:57:838.606 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476777838465 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 650 in_bytes = 48222 out_pkts = 654 out_bytes = 45828 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 185 out_bw_usage = 156 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:12:57:838.639 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:12:57:838.649 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:57:838.662 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476777838474 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:12:57:838.669 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:12:58:090.831 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:58:090.956 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476778090623 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 248 dequeues = 248 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 291 dequeues = 291 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476750099854 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476777838648 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:12:58:090.973 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:12:58:091.005 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:58:091.319 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476778090782 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 68 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 248, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:12:58:091.341 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:12:58:328.058 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:58:328.185 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476778328006 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.5825 five_min_avg = 0.61125 fifteen_min_avg = 0.54625 ] cpu_share = 0.191667 ] ] 2024-10-21 Mon 02:12:58:328.202 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:12:58:328.213 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:58:328.232 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476778328030 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.191667 used_sys_mem = 4269432198 one_min_cpuload = 0.5825 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:12:58:328.241 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:12:58:328.249 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:58:328.412 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476778328074 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 650 in_bytes = 48222 out_tpkts = 654 out_bytes = 45828 exception_packets = 48 exception_packets_allowed = 33 raw_vhost_stats= [ name = vhost0 in_pkts = 650 in_bytes = 48222 out_pkts = 654 out_bytes = 45828 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 468 in_bytes = 38016 out_pkts = 654 out_bytes = 48301 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:12:58:328.439 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:12:58:328.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:12:58:328.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476778328117 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:12:58:328.489 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:13:00:100.050 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:13:00:100.352 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:13:00:100.494 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:00:100.527 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476780100155 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:13:00:100.537 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:13:00:100.546 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:00:100.826 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476780100480 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 70 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 38, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 254, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:13:00:100.870 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:13:09:546.291 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:47612 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:13:09:589.921 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:47612 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:13:23:358.344 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:34734 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:13:23:402.202 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:34734 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:13:27:838.499 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:27:838.565 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476807838432 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:13:27:838.577 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:13:27:838.587 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:27:838.658 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476807838556 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 667 in_bytes = 49444 out_pkts = 675 out_bytes = 47166 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 325 out_bw_usage = 356 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:13:27:838.671 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:13:27:838.680 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:27:838.707 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476807838560 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:13:27:838.714 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:13:28:091.105 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:28:091.221 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476808090961 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 259 dequeues = 259 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 303 dequeues = 303 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476780100166 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476807838678 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:13:28:091.235 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:13:28:091.244 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:28:091.508 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476808091046 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 71 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 42, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 259, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:13:28:091.545 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:13:28:328.340 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:28:328.568 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476808328299 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 667 in_bytes = 49444 out_tpkts = 675 out_bytes = 47166 exception_packets = 51 exception_packets_allowed = 36 raw_vhost_stats= [ name = vhost0 in_pkts = 667 in_bytes = 49444 out_pkts = 675 out_bytes = 47166 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 441 out_bandwidth_usage = 484 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 1.20889e-15 total_out_bandwidth_utilization = 1.44199e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 223, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 266, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 486 in_bytes = 39228 out_pkts = 673 out_bytes = 49607 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:13:28:328.595 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:13:28:328.606 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:13:28:328.638 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476808328349 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:13:28:328.645 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:13:30:100.641 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:13:30:100.757 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:13:30:100.800 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:13:30:100.906 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Idle 2024-10-21 Mon 02:13:30:101.076 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:30:101.110 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476810100837 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:13:30:101.120 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:13:30:101.141 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:13:30:101.195 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:30:101.453 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476810101063 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 73 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 42, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 263, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:13:30:101.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:13:40:516.692 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:46900 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:13:40:561.812 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:46900 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:13:57:838.627 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:57:838.690 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476837838557 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:13:57:838.702 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:13:57:838.797 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:57:838.896 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476837838755 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 683 in_bytes = 50508 out_pkts = 691 out_bytes = 48122 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 283 out_bw_usage = 254 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:13:57:838.927 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:13:57:838.937 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:57:838.949 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476837838776 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:13:57:838.957 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:13:58:091.388 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:58:091.479 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476838091215 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 268 dequeues = 268 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 314 dequeues = 314 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476810100904 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476837838935 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:13:58:091.493 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:13:58:091.506 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:58:091.770 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476838091343 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 268, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:13:58:091.802 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:13:58:328.544 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:58:328.630 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476838328499 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.43125 five_min_avg = 0.565 fifteen_min_avg = 0.53375 ] cpu_share = 0.197917 ] ] 2024-10-21 Mon 02:13:58:328.641 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:13:58:328.652 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:58:328.669 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476838328523 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.197917 used_sys_mem = 4269432198 one_min_cpuload = 0.43125 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:13:58:328.677 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:13:58:328.686 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:58:328.847 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476838328559 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 683 in_bytes = 50508 out_tpkts = 691 out_bytes = 48122 exception_packets = 53 exception_packets_allowed = 38 raw_vhost_stats= [ name = vhost0 in_pkts = 683 in_bytes = 50508 out_pkts = 691 out_bytes = 48122 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 499 in_bytes = 40058 out_pkts = 689 out_bytes = 50671 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:13:58:328.872 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:13:58:328.880 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:13:58:328.912 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476838328602 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:13:58:328.919 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:14:00:101.065 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:14:00:101.357 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:14:00:101.484 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:00:101.517 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476840101155 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:14:00:101.527 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:14:00:101.536 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:00:101.872 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476840101470 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 274, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:14:00:102.600 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:14:27:838.869 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:27:839.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476867838803 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:14:27:839.025 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:14:27:839.080 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:27:839.193 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476867838931 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 690 in_bytes = 51030 out_pkts = 700 out_bytes = 48676 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 139 out_bw_usage = 147 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:14:27:839.209 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:14:27:839.221 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:27:839.238 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476867838935 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:14:27:839.248 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:14:28:091.739 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:28:091.931 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476868091563 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 279 dequeues = 279 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 326 dequeues = 326 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476840101164 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476867839219 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:14:28:091.974 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:14:28:091.990 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:28:092.497 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476868091657 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 279, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:14:28:092.543 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:14:28:328.782 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:28:329.018 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476868328738 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 690 in_bytes = 51030 out_tpkts = 700 out_bytes = 48676 raw_vhost_stats= [ name = vhost0 in_pkts = 690 in_bytes = 51030 out_pkts = 700 out_bytes = 48676 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 9.05309e-16 total_out_bandwidth_utilization = 1.14383e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 167, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 211, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 505 in_bytes = 40486 out_pkts = 696 out_bytes = 51193 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:14:28:329.048 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:14:28:329.059 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:14:28:329.090 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476868328791 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:14:28:329.482 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:14:30:101.580 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:14:30:101.687 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:14:30:101.702 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:14:30:101.830 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Idle 2024-10-21 Mon 02:14:30:101.956 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:30:101.988 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476870101778 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:14:30:101.998 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:14:30:102.006 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:14:30:102.038 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:30:102.394 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476870101943 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 79 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 283, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:14:30:102.432 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:14:57:838.847 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:57:838.914 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476897838814 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:14:57:838.926 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:14:57:839.126 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:57:839.219 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476897839112 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 698 in_bytes = 51594 out_pkts = 709 out_bytes = 49202 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 150 out_bw_usage = 140 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:14:57:839.234 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:14:57:839.243 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:57:839.255 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476897839119 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:14:57:839.262 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:14:58:092.025 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:58:092.140 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476898091800 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 288 dequeues = 288 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 337 dequeues = 337 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476870101828 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476897839242 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:14:58:092.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:14:58:092.187 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:58:092.453 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476898091941 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 80 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 288, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:14:58:093.140 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:14:58:329.073 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:58:329.213 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476898329003 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.28 five_min_avg = 0.4975 fifteen_min_avg = 0.5125 ] cpu_share = 0.195833 ] ] 2024-10-21 Mon 02:14:58:329.229 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:14:58:329.242 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:58:329.261 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476898329027 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.195833 used_sys_mem = 4269432198 one_min_cpuload = 0.28 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:14:58:329.293 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:14:58:329.302 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:58:329.457 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476898329070 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 698 in_bytes = 51594 out_tpkts = 709 out_bytes = 49202 raw_vhost_stats= [ name = vhost0 in_pkts = 698 in_bytes = 51594 out_pkts = 709 out_bytes = 49202 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 511 in_bytes = 40886 out_pkts = 704 out_bytes = 51757 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:14:58:329.473 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:14:58:329.481 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:14:58:329.514 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729476898329131 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:14:58:329.521 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:15:00:101.987 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:15:00:102.261 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:15:00:102.393 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:00:102.454 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476900102084 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:15:00:102.469 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:15:00:102.484 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:00:102.768 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729476900102370 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 82 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 294, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:15:00:102.804 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:15:26:505.550 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:39278 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:15:26:549.877 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:39278 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:15:27:840.518 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:27:840.599 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476927840473 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:15:27:840.614 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:15:27:840.660 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:27:840.774 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476927840639 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 713 in_bytes = 52736 out_pkts = 724 out_bytes = 50188 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 304 out_bw_usage = 262 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:15:27:840.806 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:15:27:840.816 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:27:840.828 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476927840650 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:15:27:840.835 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:15:28:092.290 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:28:092.415 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476928092138 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 299 dequeues = 299 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 349 dequeues = 349 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476900102093 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476927840814 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:15:28:092.430 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:15:28:092.439 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:28:092.729 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476928092237 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 83 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 299, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:15:28:092.777 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:15:28:329.287 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:28:329.507 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476928329244 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 713 in_bytes = 52736 out_tpkts = 724 out_bytes = 50188 exception_packets = 54 exception_packets_allowed = 39 raw_vhost_stats= [ name = vhost0 in_pkts = 713 in_bytes = 52736 out_pkts = 724 out_bytes = 50188 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 9.1073e-16 total_out_bandwidth_utilization = 1.23057e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 168, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 227, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 523 in_bytes = 41746 out_pkts = 719 out_bytes = 52899 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:15:28:329.534 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:15:28:329.546 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:15:28:329.585 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476928329297 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:15:28:329.593 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:15:30:102.405 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:15:30:102.474 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:15:30:102.485 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:15:30:102.574 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Idle 2024-10-21 Mon 02:15:30:102.740 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:30:102.773 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476930102515 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:15:30:102.782 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:15:30:102.802 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:15:30:102.835 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:30:103.210 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476930102720 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 85 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 303, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:15:30:103.241 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:15:57:840.743 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:57:840.815 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476957840695 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:15:57:840.828 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:15:57:840.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:57:841.107 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476957840940 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 722 in_bytes = 53294 out_pkts = 733 out_bytes = 50638 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 148 out_bw_usage = 120 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:15:57:841.137 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:15:57:841.148 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:57:841.160 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476957840981 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:15:57:841.167 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:15:58:092.645 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:58:092.762 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476958092391 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 308 dequeues = 308 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 360 dequeues = 360 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476930102572 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729476957841146 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:15:58:092.779 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:15:58:092.790 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:58:093.088 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729476958092559 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 86 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 308, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:15:58:093.120 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:15:58:329.488 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:58:329.585 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476958329447 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.26375 five_min_avg = 0.4525 fifteen_min_avg = 0.49625 ] cpu_share = 0.195833 ] ] 2024-10-21 Mon 02:15:58:329.598 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:15:58:329.608 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:58:329.628 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476958329469 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.195833 used_sys_mem = 4269432198 one_min_cpuload = 0.26375 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:15:58:329.636 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:15:58:329.644 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:58:329.828 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476958329508 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 722 in_bytes = 53294 out_tpkts = 733 out_bytes = 50638 exception_packets = 55 exception_packets_allowed = 40 raw_vhost_stats= [ name = vhost0 in_pkts = 722 in_bytes = 53294 out_pkts = 733 out_bytes = 50638 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 529 in_bytes = 42070 out_pkts = 728 out_bytes = 53457 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:15:58:329.859 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:15:58:329.871 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:15:58:329.902 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476958329554 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:15:58:329.910 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:16:00:102.736 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:16:00:103.012 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:16:00:103.183 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:00:103.230 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476960102840 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:16:00:103.242 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:16:00:103.255 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:00:103.781 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729476960103162 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 88 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 314, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:16:00:103.825 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:16:27:841.105 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:27:841.277 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476987841032 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 733 in_bytes = 54080 out_pkts = 745 out_bytes = 51386 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 209 out_bw_usage = 199 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:16:27:841.310 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:16:27:841.325 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:27:841.344 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476987841056 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:16:27:841.355 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:16:27:841.366 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:27:841.385 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729476987841086 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:16:27:841.395 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:16:28:092.948 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:28:093.143 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476988092772 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 319 dequeues = 319 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 372 dequeues = 372 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729476960102851 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729476987841364 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:16:28:093.178 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:16:28:093.188 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:28:093.439 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729476988092884 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 89 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 319, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:16:28:093.459 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:16:28:329.903 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:28:330.123 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476988329856 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 733 in_bytes = 54080 out_tpkts = 745 out_bytes = 51386 exception_packets = 56 exception_packets_allowed = 41 raw_vhost_stats= [ name = vhost0 in_pkts = 733 in_bytes = 54080 out_pkts = 745 out_bytes = 51386 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 6.83047e-16 total_out_bandwidth_utilization = 9.70361e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 126, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 179, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 538 in_bytes = 42692 out_pkts = 739 out_bytes = 54243 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:16:28:330.153 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:16:28:330.164 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:16:28:330.197 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476988329911 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:16:28:330.204 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:16:30:104.372 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:16:30:104.483 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:16:30:104.498 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:16:30:104.914 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Idle 2024-10-21 Mon 02:16:30:105.147 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:30:105.199 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476990104544 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:16:30:105.213 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:16:30:105.226 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:16:30:105.265 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:30:105.782 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729476990105120 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 91 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 323, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:16:30:105.826 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:16:57:842.561 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:57:842.629 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477017842512 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:16:57:842.655 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:16:57:842.722 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:57:842.824 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477017842675 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 740 in_bytes = 54554 out_pkts = 752 out_bytes = 51752 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:16:57:842.838 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:16:57:842.846 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:57:842.858 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477017842703 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:16:57:842.865 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:16:58:093.265 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:58:093.357 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477018093100 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 328 dequeues = 328 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 383 dequeues = 383 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729476990104910 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477017842845 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:16:58:093.387 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:16:58:093.397 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:58:093.668 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477018093220 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 92 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 328, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:16:58:093.706 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:16:58:330.184 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:58:330.273 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477018330089 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.26875 five_min_avg = 0.42 fifteen_min_avg = 0.4825 ] cpu_share = 0.19375 ] ] 2024-10-21 Mon 02:16:58:330.285 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:16:58:330.296 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:58:330.313 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477018330136 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.19375 used_sys_mem = 4269432198 one_min_cpuload = 0.26875 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:16:58:330.321 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:16:58:330.329 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:58:330.506 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477018330173 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 740 in_bytes = 54554 out_tpkts = 752 out_bytes = 51752 raw_vhost_stats= [ name = vhost0 in_pkts = 740 in_bytes = 54554 out_pkts = 752 out_bytes = 51752 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 542 in_bytes = 42932 out_pkts = 746 out_bytes = 54717 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:16:58:330.529 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:16:58:330.542 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:16:58:330.573 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477018330212 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:16:58:330.580 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:17:00:105.125 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:17:00:105.419 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:17:00:105.540 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:00:105.571 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477020105225 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:17:00:105.580 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:17:00:105.588 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:00:105.876 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477020105523 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 94 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 46, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 334, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:17:00:105.914 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:17:19:744.681 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:60056 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:17:19:789.934 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:60056 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:17:27:849.083 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:27:849.138 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477047849012 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:17:27:849.149 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:17:27:849.157 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:27:849.230 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477047849054 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 776 in_bytes = 57434 out_pkts = 788 out_bytes = 54560 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 768 out_bw_usage = 748 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:17:27:849.242 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:17:27:849.250 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:27:849.273 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477047849057 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:17:27:849.281 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:17:28:095.589 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:28:095.760 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477048095411 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 339 dequeues = 339 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 395 dequeues = 395 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729477020105237 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477047849248 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:17:28:095.785 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:17:28:095.801 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:28:096.261 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477048095550 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 95 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 36 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 48, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 339, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:17:28:096.308 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:17:28:331.166 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:28:331.375 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477048331117 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 776 in_bytes = 57434 out_tpkts = 788 out_bytes = 54560 exception_packets = 58 exception_packets_allowed = 43 raw_vhost_stats= [ name = vhost0 in_pkts = 776 in_bytes = 57434 out_pkts = 788 out_bytes = 54560 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 2.10877e-15 total_out_bandwidth_utilization = 2.42319e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 389, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 447, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 575 in_bytes = 45614 out_pkts = 782 out_bytes = 57597 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:17:28:333.104 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:17:28:333.120 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:17:28:333.157 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477048331430 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:17:28:333.165 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:17:30:105.765 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:17:30:105.844 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:17:30:105.855 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:17:30:107.342 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Idle 2024-10-21 Mon 02:17:30:107.565 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:30:107.609 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477050105903 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:17:30:107.619 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:17:30:107.629 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:17:30:107.666 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:30:108.080 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477050107544 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 97 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 36 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 48, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 343, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:17:30:108.121 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:17:35:089.760 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:48098 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:17:35:134.300 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:48098 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:17:52:387.026 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:43188 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:17:52:434.251 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:43188 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:17:57:850.502 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:57:850.564 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477077850434 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:17:57:850.575 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:17:57:850.669 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:57:850.751 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477077850658 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 880 in_bytes = 66640 out_pkts = 892 out_bytes = 63658 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2454 out_bw_usage = 2426 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:17:57:850.780 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:17:57:850.789 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:57:850.800 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477077850667 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:17:57:850.807 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:17:58:095.831 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:58:095.927 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477078095667 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 348 dequeues = 348 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 406 dequeues = 406 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729477050107336 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477077850787 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:17:58:095.941 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:17:58:095.949 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:58:096.214 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477078095802 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 98 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 348, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:17:58:096.250 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:17:58:331.361 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:58:331.483 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477078331315 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.55375 five_min_avg = 0.48125 fifteen_min_avg = 0.5 ] cpu_share = 0.210417 ] ] 2024-10-21 Mon 02:17:58:331.500 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:17:58:331.512 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:58:331.529 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477078331337 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.210417 used_sys_mem = 4269432198 one_min_cpuload = 0.55375 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:17:58:331.538 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:17:58:331.550 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:58:331.709 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477078331371 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 880 in_bytes = 66640 out_tpkts = 892 out_bytes = 63658 raw_vhost_stats= [ name = vhost0 in_pkts = 880 in_bytes = 66640 out_pkts = 892 out_bytes = 63658 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 676 in_bytes = 54586 out_pkts = 886 out_bytes = 66803 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:17:58:331.737 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:17:58:331.745 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:17:58:331.777 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477078331429 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:17:58:331.788 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:18:00:107.521 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:18:00:107.884 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:18:00:108.048 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:00:108.116 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477080107641 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:18:00:108.130 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:18:00:108.144 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:00:108.640 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477080108025 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 100 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 354, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:18:00:108.699 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:18:27:850.631 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:27:850.707 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477107850568 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:18:27:850.734 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:18:27:850.744 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:27:850.816 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477107850708 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 890 in_bytes = 67288 out_pkts = 905 out_bytes = 64352 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 172 out_bw_usage = 185 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:18:27:850.828 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:18:27:850.836 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:27:850.851 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477107850714 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:18:27:850.858 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:18:28:096.146 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:28:096.268 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477108095981 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 359 dequeues = 359 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 418 dequeues = 418 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729477080107652 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477107850835 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:18:28:096.295 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:18:28:096.306 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:28:096.560 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477108096095 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 101 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 359, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:18:28:096.580 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:18:28:331.688 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:28:332.043 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477108331623 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 890 in_bytes = 67288 out_tpkts = 905 out_bytes = 64352 exception_packets = 61 exception_packets_allowed = 46 raw_vhost_stats= [ name = vhost0 in_pkts = 890 in_bytes = 67288 out_pkts = 905 out_bytes = 64352 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 424 out_bandwidth_usage = 478 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 6.89553e-15 total_out_bandwidth_utilization = 7.18284e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1272, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1325, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 686 in_bytes = 55154 out_pkts = 898 out_bytes = 67535 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:18:28:332.100 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:18:28:332.119 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:18:28:332.171 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477108331692 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:18:28:332.191 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:18:30:108.063 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:18:30:108.155 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:18:30:108.190 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:18:30:108.287 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Idle 2024-10-21 Mon 02:18:30:108.395 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:30:108.428 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477110108226 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:18:30:108.438 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:18:30:108.448 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:18:30:108.479 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:30:108.764 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477110108383 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 103 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 363, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:18:30:109.733 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:18:57:850.722 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:57:850.789 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477137850685 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:18:57:850.800 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:18:57:850.973 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:57:851.137 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477137850932 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 897 in_bytes = 67762 out_pkts = 914 out_bytes = 64858 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 134 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:18:57:851.157 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:18:57:851.170 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:57:851.187 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477137850948 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:18:57:851.197 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:18:58:096.454 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:58:096.560 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477138096227 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 368 dequeues = 368 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 429 dequeues = 429 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729477110108285 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477137851167 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:18:58:096.573 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:18:58:096.581 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:58:096.861 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477138096408 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 104 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 368, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:18:58:096.882 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:18:58:331.885 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:58:331.983 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477138331845 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.375 five_min_avg = 0.4425 fifteen_min_avg = 0.485 ] cpu_share = 0.19375 ] ] 2024-10-21 Mon 02:18:58:331.995 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:18:58:332.020 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:58:332.041 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477138331867 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.19375 used_sys_mem = 4269432198 one_min_cpuload = 0.375 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:18:58:332.049 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:18:58:332.057 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:58:332.219 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477138331902 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 897 in_bytes = 67762 out_tpkts = 914 out_bytes = 64858 raw_vhost_stats= [ name = vhost0 in_pkts = 897 in_bytes = 67762 out_pkts = 914 out_bytes = 64858 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 692 in_bytes = 55534 out_pkts = 905 out_bytes = 68009 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:18:58:332.235 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:18:58:332.246 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:18:58:332.278 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477138331943 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:18:58:332.284 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:19:00:108.449 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:19:00:108.774 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:19:00:108.891 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:00:108.921 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477140108594 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:19:00:108.930 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:19:00:108.939 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:00:109.235 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477140108876 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 106 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 374, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:19:00:109.269 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:19:27:850.761 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:27:850.815 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477167850727 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:19:27:850.827 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:19:27:850.969 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:27:851.052 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477167850956 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 908 in_bytes = 68500 out_pkts = 925 out_bytes = 65488 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 196 out_bw_usage = 168 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:19:27:851.068 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:19:27:851.090 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:27:851.102 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477167850966 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:19:27:851.109 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:19:28:096.741 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:28:096.861 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477168096594 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 379 dequeues = 379 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 441 dequeues = 441 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729477140108605 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477167851088 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:19:28:096.876 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:19:28:096.885 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:28:097.156 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477168096697 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 107 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 379, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:19:28:097.193 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:19:28:332.133 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:28:332.378 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477168332086 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 908 in_bytes = 68500 out_tpkts = 925 out_bytes = 65488 exception_packets = 62 exception_packets_allowed = 47 raw_vhost_stats= [ name = vhost0 in_pkts = 908 in_bytes = 68500 out_pkts = 925 out_bytes = 65488 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 6.34258e-16 total_out_bandwidth_utilization = 8.72783e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 117, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 161, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 700 in_bytes = 56038 out_pkts = 916 out_bytes = 68747 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:19:28:332.410 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:19:28:332.421 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:19:28:332.453 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477168332147 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:19:28:332.461 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:19:30:108.882 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:19:30:108.958 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:19:30:109.054 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:19:30:109.151 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Idle 2024-10-21 Mon 02:19:30:109.263 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:30:109.295 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477170109088 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:19:30:109.304 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:19:30:109.324 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:19:30:109.351 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:30:109.633 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477170109248 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 109 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 383, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:19:30:109.661 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:19:57:851.080 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:57:851.148 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477197851008 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:19:57:851.160 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:19:57:851.170 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:57:851.266 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477197851157 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 916 in_bytes = 69044 out_pkts = 932 out_bytes = 65854 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 145 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:19:57:851.300 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:19:57:851.311 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:57:851.323 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477197851162 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:19:57:851.331 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:19:58:097.134 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:58:097.283 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477198096870 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 388 dequeues = 388 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 452 dequeues = 452 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729477170109149 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477197851310 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:19:58:097.304 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:19:58:097.318 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:58:097.866 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477198097077 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 110 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 388, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:19:58:097.924 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:19:58:332.346 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:58:332.435 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477198332301 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.3075 five_min_avg = 0.40875 fifteen_min_avg = 0.47 ] cpu_share = 0.191667 ] ] 2024-10-21 Mon 02:19:58:332.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:19:58:332.461 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:58:332.478 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477198332325 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.191667 used_sys_mem = 4269432198 one_min_cpuload = 0.3075 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:19:58:332.485 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:19:58:332.493 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:58:332.642 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477198332361 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 916 in_bytes = 69044 out_tpkts = 932 out_bytes = 65854 raw_vhost_stats= [ name = vhost0 in_pkts = 916 in_bytes = 69044 out_pkts = 932 out_bytes = 65854 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 704 in_bytes = 56278 out_pkts = 924 out_bytes = 69291 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:19:58:332.677 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:19:58:332.685 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:19:58:332.715 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477198332406 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:19:58:332.722 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:20:00:109.330 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:20:00:109.747 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:20:00:109.918 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:00:109.985 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477200109511 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:20:00:109.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:20:00:110.012 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:00:110.543 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477200109896 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 112 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 394, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:20:00:110.591 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:20:27:851.297 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:27:851.375 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477227851255 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:20:27:851.390 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:20:27:851.495 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:27:851.636 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477227851466 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 939 in_bytes = 70862 out_pkts = 955 out_bytes = 67564 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 484 out_bw_usage = 456 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:20:27:851.689 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:20:27:851.703 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:27:851.720 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477227851477 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:20:27:851.730 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:20:28:097.519 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:28:097.744 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477228097315 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 399 dequeues = 399 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 464 dequeues = 464 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729477200109523 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477227851701 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:20:28:097.788 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:20:28:097.804 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:28:098.331 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477228097455 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 113 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 399, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:20:28:098.375 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:20:28:332.584 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:28:332.807 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477228332539 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 939 in_bytes = 70862 out_tpkts = 955 out_bytes = 67564 exception_packets = 63 exception_packets_allowed = 48 raw_vhost_stats= [ name = vhost0 in_pkts = 939 in_bytes = 70862 out_pkts = 955 out_bytes = 67564 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 1.31731e-15 total_out_bandwidth_utilization = 1.7022e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 243, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 314, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 724 in_bytes = 57862 out_pkts = 947 out_bytes = 71109 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:20:28:332.832 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:20:28:332.843 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:20:28:332.875 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477228332593 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:20:28:332.883 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:20:30:109.944 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:20:30:110.032 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:20:30:110.045 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:20:30:110.487 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Idle 2024-10-21 Mon 02:20:30:110.708 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:30:110.749 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477230110084 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:20:30:110.759 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:20:30:110.768 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:20:30:110.817 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:30:111.131 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477230110612 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 115 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 403, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:20:30:111.165 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:20:57:851.805 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:57:851.885 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477257851730 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:20:57:851.915 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:20:57:852.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:57:852.156 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477257851976 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 951 in_bytes = 71786 out_pkts = 968 out_bytes = 68450 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 246 out_bw_usage = 236 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:20:57:852.175 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:20:57:852.189 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:57:852.206 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477257851983 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:20:57:852.217 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:20:58:097.837 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:58:098.029 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477258097609 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 408 dequeues = 408 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 475 dequeues = 475 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729477230110483 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477257852186 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:20:58:098.075 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:20:58:098.091 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:58:098.603 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477258097782 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 116 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 408, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:20:58:098.662 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:20:58:332.804 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:58:332.895 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477258332757 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.26875 five_min_avg = 0.37625 fifteen_min_avg = 0.455 ] cpu_share = 0.19375 ] ] 2024-10-21 Mon 02:20:58:332.908 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:20:58:332.919 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:58:332.936 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477258332780 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.19375 used_sys_mem = 4269432198 one_min_cpuload = 0.26875 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:20:58:332.945 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:20:58:332.953 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:58:333.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477258332829 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 951 in_bytes = 71786 out_tpkts = 968 out_bytes = 68450 exception_packets = 64 exception_packets_dropped = 16 raw_vhost_stats= [ name = vhost0 in_pkts = 951 in_bytes = 71786 out_pkts = 968 out_bytes = 68450 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 734 in_bytes = 58622 out_pkts = 959 out_bytes = 72033 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:20:58:333.140 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:20:58:333.148 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:20:58:333.187 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477258332871 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:20:58:333.194 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:21:00:110.769 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:21:00:111.012 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:21:00:111.125 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:00:111.157 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477260110850 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:21:00:111.167 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:21:00:111.175 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:00:111.430 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477260111111 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 118 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 52, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 414, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:21:00:111.463 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:21:14:724.824 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:59872 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:21:14:774.047 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:59872 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:21:27:851.893 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:27:851.982 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477287851815 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:21:27:851.998 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:21:27:852.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:27:852.221 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477287852048 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 965 in_bytes = 72766 out_pkts = 982 out_bytes = 69449 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 261 out_bw_usage = 266 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:21:27:852.238 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:21:27:852.250 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:27:852.262 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477287852077 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:21:27:852.285 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:21:28:098.095 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:28:098.230 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477288097970 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 419 dequeues = 419 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 487 dequeues = 487 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729477260110862 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477287852248 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:21:28:098.246 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:21:28:098.256 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:28:098.610 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477288098054 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 119 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 419, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:21:28:098.660 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:21:28:333.059 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:28:333.433 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477288333015 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 965 in_bytes = 72766 out_tpkts = 982 out_bytes = 69449 exception_packets = 65 exception_packets_allowed = 49 raw_vhost_stats= [ name = vhost0 in_pkts = 965 in_bytes = 72766 out_pkts = 982 out_bytes = 69449 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 1.17636e-15 total_out_bandwidth_utilization = 1.37152e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 217, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 253, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 745 in_bytes = 59495 out_pkts = 973 out_bytes = 73013 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:21:28:333.469 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:21:28:333.484 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:21:28:333.536 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477288333059 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:21:28:333.546 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:21:30:111.174 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:21:30:111.271 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:21:30:111.313 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:21:30:111.423 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Idle 2024-10-21 Mon 02:21:30:111.535 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:30:111.567 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477290111351 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:21:30:111.577 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:21:30:111.585 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:21:30:111.615 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:30:111.899 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477290111521 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 121 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 423, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:21:30:111.939 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:21:57:851.869 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:57:851.929 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477317851830 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:21:57:851.942 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:21:57:852.083 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:57:852.242 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477317852055 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 975 in_bytes = 73414 out_pkts = 992 out_bytes = 69989 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 172 out_bw_usage = 144 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:21:57:852.281 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:21:57:852.297 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:57:852.315 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477317852062 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:21:57:852.327 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:21:58:098.352 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:58:098.457 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477318098190 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 428 dequeues = 428 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 498 dequeues = 498 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729477290111421 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477317852294 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:21:58:098.471 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:21:58:098.493 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:58:098.772 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: SANDESH: Send FAILED: 1729477318098310 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 122 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 428, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:21:58:098.803 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575639480064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:21:58:333.267 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:58:333.362 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477318333223 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.35125 five_min_avg = 0.38125 fifteen_min_avg = 0.45125 ] cpu_share = 0.197884 ] ] 2024-10-21 Mon 02:21:58:333.374 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:21:58:333.384 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:58:333.432 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477318333245 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.197884 used_sys_mem = 4269432198 one_min_cpuload = 0.35125 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:21:58:333.447 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:21:58:333.460 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:58:333.613 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477318333278 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 975 in_bytes = 73414 out_tpkts = 992 out_bytes = 69989 exception_packets = 66 exception_packets_allowed = 50 raw_vhost_stats= [ name = vhost0 in_pkts = 975 in_bytes = 73414 out_pkts = 992 out_bytes = 69989 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 752 in_bytes = 59909 out_pkts = 983 out_bytes = 73661 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:21:58:333.651 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:21:58:333.659 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:21:58:333.690 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477318333321 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:21:58:333.697 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:22:00:111.573 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:22:00:111.847 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:22:00:111.965 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:00:111.997 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477320111685 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:22:00:112.019 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:22:00:112.029 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:00:112.290 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477320111950 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 124 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 57 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 434, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:22:00:112.311 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:22:27:853.733 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:27:853.813 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477347853654 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:22:27:853.830 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:22:27:853.844 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:27:853.963 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477347853822 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 982 in_bytes = 73888 out_pkts = 999 out_bytes = 70355 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:22:27:853.999 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:22:27:854.013 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:27:854.031 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: SANDESH: Send FAILED: 1729477347853831 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:22:27:854.042 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575635281664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:22:28:098.700 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:28:098.967 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477348098543 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 439 dequeues = 439 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 510 dequeues = 510 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729477320111693 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477347854011 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:22:28:098.989 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:22:28:099.013 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:28:099.265 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477348098653 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 125 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 57 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 439, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:22:28:099.283 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:22:28:333.496 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:28:333.719 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477348333433 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 982 in_bytes = 73888 out_tpkts = 999 out_bytes = 70355 raw_vhost_stats= [ name = vhost0 in_pkts = 982 in_bytes = 73888 out_pkts = 999 out_bytes = 70355 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 4.71628e-16 total_out_bandwidth_utilization = 8.07731e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 87, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 149, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 756 in_bytes = 60149 out_pkts = 990 out_bytes = 74135 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:22:28:333.746 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:22:28:333.757 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:22:28:333.788 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477348333475 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:22:28:333.795 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:22:30:112.045 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:22:30:112.136 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvConnectTimerExpired in state Connect 2024-10-21 Mon 02:22:30:112.150 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Connect : EvConnectTimerExpired 2024-10-21 Mon 02:22:30:112.268 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Idle 2024-10-21 Mon 02:22:30:112.411 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:30:112.450 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477350112196 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:22:30:112.705 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:22:30:112.728 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvTcpDeleteSession in state Idle 2024-10-21 Mon 02:22:30:112.794 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:30:113.286 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477350112395 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 127 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 443, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:22:30:113.330 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:22:51:788.906 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:56692 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:22:51:837.816 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:56692 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:22:57:853.844 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:57:854.077 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477377853766 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:22:57:854.101 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:22:57:854.110 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:57:854.185 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477377853949 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 998 in_bytes = 74940 out_pkts = 1014 out_bytes = 71490 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 280 out_bw_usage = 302 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:22:57:854.197 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:22:57:854.206 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:57:854.234 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: SANDESH: Send FAILED: 1729477377853959 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:22:57:854.243 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575643678464, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:22:58:098.937 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:58:099.077 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477378098772 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 448 dequeues = 448 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 521 dequeues = 521 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1729477350112265 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477377854204 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:22:58:099.112 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:22:58:099.123 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:58:099.386 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477378098892 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 128 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 56, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 448, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:22:58:099.406 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:22:58:333.676 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:58:333.783 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477378333634 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25638656 used = 0 free = 1494150702 buffers = 32730 cached = 25502368 node_type = ] meminfo= [ virt = 32730 peakvirt = 1480079768 res = 32730 ] cpuload= [ one_min_avg = 0.31125 five_min_avg = 0.36125 fifteen_min_avg = 0.44 ] cpu_share = 0.19375 ] ] 2024-10-21 Mon 02:22:58:333.795 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:22:58:333.805 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:58:333.822 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477378333656 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 cpu_info= [ [ [ mem_virt = 32730 cpu_share = 0.19375 used_sys_mem = 4269432198 one_min_cpuload = 0.31125 mem_res = 32730 ], ] ] ] 2024-10-21 Mon 02:22:58:333.831 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2024-10-21 Mon 02:22:58:333.855 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:58:334.004 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477378333690 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 in_tpkts = 998 in_bytes = 74940 out_tpkts = 1014 out_bytes = 71490 exception_packets = 68 exception_packets_allowed = 52 raw_vhost_stats= [ name = vhost0 in_pkts = 998 in_bytes = 74940 out_pkts = 1014 out_bytes = 71490 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 768 in_bytes = 61158 out_pkts = 1006 out_bytes = 75187 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2024-10-21 Mon 02:22:58:334.016 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2024-10-21 Mon 02:22:58:334.028 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Processing scm::EvSandeshSend in state Idle 2024-10-21 Mon 02:22:58:334.058 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: SANDESH: Send FAILED: 1729477378333751 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.30 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2024-10-21 Mon 02:22:58:334.065 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575647876864, Pid 1095487]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2024-10-21 Mon 02:23:00:112.459 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvIdleHoldTimerExpired in state Idle 2024-10-21 Mon 02:23:00:112.810 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Connect : Start Connect timer 10.0.0.30:8086 2024-10-21 Mon 02:23:00:112.947 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:23:00:112.995 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477380112567 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.30 server_addrs= [ [ (*_iter6) = 10.0.0.30:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.30:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2024-10-21 Mon 02:23:00:113.006 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2024-10-21 Mon 02:23:00:113.015 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:23:00:113.273 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: SANDESH: Send FAILED: 1729477380112930 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 130 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 60 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 56, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 454, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-21 Mon 02:23:00:113.307 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575656273664, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:23:05:521.415 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:49904 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2024-10-21 Mon 02:23:05:565.958 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575805192512, Pid 1095487]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8085::10.0.0.30:49904 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2024-10-21 Mon 02:23:27:855.068 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:23:27:855.132 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477407854923 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2024-10-21 Mon 02:23:27:855.142 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2024-10-21 Mon 02:23:27:855.150 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:23:27:855.220 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477407855040 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 1076 in_bytes = 81825 out_pkts = 1100 out_bytes = 79107 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 1836 out_bw_usage = 2031 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 2 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2024-10-21 Mon 02:23:27:855.233 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2024-10-21 Mon 02:23:27:855.241 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:23:27:855.263 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: SANDESH: Send FAILED: 1729477407855044 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-439-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2024-10-21 Mon 02:23:27:855.270 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575660472064, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2024-10-21 Mon 02:23:28:101.775 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:23:28:101.888 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477408101656 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 459 dequeues = 459 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 533 dequeues = 533 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1729477380112576 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477407855240 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2024-10-21 Mon 02:23:28:101.901 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2024-10-21 Mon 02:23:28:101.910 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: Processing scm::EvSandeshSend in state Connect 2024-10-21 Mon 02:23:28:105.882 UTC cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140575631083264, Pid 1095487]: SANDESH: Send FAILED: 1729477408101748 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1095487 http_port = 8085 start_time = 1729476178082920 collector_name = collector_ip = 10.0.0.30:8086 collector_list= [ [ (*_iter40) = 10.0.0.30:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 56 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 131 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 46 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 60 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 58, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 459, _iter59->first = sent _iter59->second = 0, ] ] ]