++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/config-database-nodemgr ++ CONTAINER_LOG_DIR=/var/log/contrail/config-database-nodemgr ++ mkdir -p /var/log/contrail/config-database-nodemgr ++ log_file=/var/log/contrail/config-database-nodemgr/console.log ++ touch /var/log/contrail/config-database-nodemgr/console.log ++ chmod 600 /var/log/contrail/config-database-nodemgr/console.log ++ exec +++ tee -a /var/log/contrail/config-database-nodemgr/console.log +++ date ++ echo 'INFO: =================== Thu Aug 28 05:19:51 PM UTC 2025 ===================' INFO: =================== Thu Aug 28 05:19:51 PM UTC 2025 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' +++++ ip route get 1 ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ ip addr show dev ens3 +++ cut -d / -f 1 +++ grep 'inet ' +++ head -n 1 +++ awk '{print $2}' ++ DEFAULT_LOCAL_IP=10.0.0.50 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=0 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 0 ]] ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=kubernetes ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=no-auth ++ AUTH_MODE=noauth ++ AUTH_PARAMS= ++ SSL_ENABLE=False ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.50 ++ ANALYTICS_ALARM_ENABLE=False ++ ANALYTICS_SNMP_ENABLE=False ++ ANALYTICSDB_ENABLE=False ++ ANALYTICS_NODES=10.0.0.50 ++ ANALYTICSDB_NODES=10.0.0.50 ++ ANALYTICS_SNMP_NODES=10.0.0.50 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:8081 ' +++ '[' -n '10.0.0.50:8081 ' ']' +++ echo 10.0.0.50:8081 ++ ANALYTICS_SERVERS=10.0.0.50:8081 +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9042 ' +++ '[' -n '10.0.0.50:9042 ' ']' +++ echo 10.0.0.50:9042 ++ ANALYTICSDB_CQL_SERVERS=10.0.0.50:9042 ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.50 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:8086 ' +++ '[' -n '10.0.0.50:8086 ' ']' +++ echo 10.0.0.50:8086 ++ COLLECTOR_SERVERS=10.0.0.50:8086 ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9041 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7201 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.50 ++ CONFIGDB_NODES=10.0.0.50 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:8082 ' +++ '[' -n '10.0.0.50:8082 ' ']' +++ echo 10.0.0.50:8082 ++ CONFIG_SERVERS=10.0.0.50:8082 +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9161 ' +++ '[' -n '10.0.0.50:9161 ' ']' +++ echo 10.0.0.50:9161 ++ CONFIGDB_SERVERS=10.0.0.50:9161 +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9041 ' +++ '[' -n '10.0.0.50:9041 ' ']' +++ echo 10.0.0.50:9041 ++ CONFIGDB_CQL_SERVERS=10.0.0.50:9041 ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=False ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=False ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=False ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.0.0.50 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.0.0.50 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.50 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+=10.0.0.50:2181, +++ '[' -n 10.0.0.50:2181, ']' +++ echo 10.0.0.50:2181 ++ ZOOKEEPER_SERVERS=10.0.0.50:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:2181 ' +++ '[' -n '10.0.0.50:2181 ' ']' +++ echo 10.0.0.50:2181 ++ ZOOKEEPER_SERVERS_SPACE_DELIM=10.0.0.50:2181 ++ RABBITMQ_NODES=10.0.0.50 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+=10.0.0.50:5673, +++ '[' -n 10.0.0.50:5673, ']' +++ echo 10.0.0.50:5673 ++ RABBITMQ_SERVERS=10.0.0.50:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=False ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:6379 ' +++ '[' -n '10.0.0.50:6379 ' ']' +++ echo 10.0.0.50:6379 ++ REDIS_SERVERS=10.0.0.50:6379 ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=False ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.50 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9092 ' +++ '[' -n '10.0.0.50:9092 ' ']' +++ echo 10.0.0.50:9092 ++ KAFKA_SERVERS=10.0.0.50:9092 ++ KAFKA_SSL_ENABLE=False ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/ ++ KEYSTONE_AUTH_HOST=127.0.0.1 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.50 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.50 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=False ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=False ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=False ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ / == \/\v\2\.\0 ]] ++ [[ kubernetes == \o\p\e\n\s\t\a\c\k ]] ++ [[ noauth == \k\e\y\s\t\o\n\e ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ noauth == \k\e\y\s\t\o\n\e ]] + cat + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + [[ config-database == \t\o\r\a\g\e\n\t ]] + export NODEMGR_TYPE=contrail-config-database + NODEMGR_TYPE=contrail-config-database + NODEMGR_NAME=contrail-config-database-nodemgr ++ tr - _ ++ echo CONFIG-DATABASE + ntype=CONFIG_DATABASE + [[ CONFIG_DATABASE == \V\R\O\U\T\E\R ]] + [[ CONFIG_DATABASE == \D\A\T\A\B\A\S\E ]] + [[ CONFIG_DATABASE == \C\O\N\F\I\G\_\D\A\T\A\B\A\S\E ]] + htype=CONFIGDB + [[ CONFIG_DATABASE == \C\O\N\T\R\O\L ]] ++ get_listen_ip_for_node CONFIGDB +++ cut -d ' ' -f 1 +++ find_my_ip_and_order_for_node CONFIGDB +++ local server_typ=CONFIGDB_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.50 +++ local servers=10.0.0.50 +++ local server_list= +++ IFS=, +++ read -ra server_list ++++ tr '\n' , ++++ get_local_ips ++++ uniq ++++ grep -vi host ++++ cat /proc/net/fib_trie ++++ awk '/32 host/ { print f } {f=$2}' ++++ sort +++ local local_ips=,10.0.0.50,10.145.217.1,127.0.0.1,172.17.0.1,252.50.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.50'\''))' +++ local server_ip=10.0.0.50 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.50 ]] +++ [[ ,10.0.0.50,10.145.217.1,127.0.0.1,172.17.0.1,252.50.0.1,, =~ ,10\.0\.0\.50, ]] +++ echo 10.0.0.50 1 +++ return ++ local ip=10.0.0.50 ++ [[ -z 10.0.0.50 ]] ++ echo 10.0.0.50 + hostip=10.0.0.50 + introspect_ip=0.0.0.0 + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + mkdir -p /etc/contrail + cat ++ format_boolean False ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ echo False + '[' -n '' ']' + cat + add_ini_params_from_env CONFIG_DATABASE_NODEMGR /etc/contrail/contrail-config-database-nodemgr.conf + local service_name=CONFIG_DATABASE_NODEMGR + local cfg_path=/etc/contrail/contrail-config-database-nodemgr.conf + local delim=__ ++ set -o posix ++ set ++ grep '^CONFIG_DATABASE_NODEMGR__.*__.*=.*$' ++ sort ++ cut -d = -f 1 ++ sed 's/^CONFIG_DATABASE_NODEMGR__//g' + local vars=DEFAULTS__minimum_diskGB + local section= + for var in $vars + local var_name=CONFIG_DATABASE_NODEMGR__DEFAULTS__minimum_diskGB + local val=4 ++ echo DEFAULTS__minimum_diskGB ++ sed s/DASH/-/g + local var_unsubst=DEFAULTS__minimum_diskGB ++ echo DEFAULTS__minimum_diskGB ++ sed 's/^\(.*\)__.*$/\1/' + local var_section=DEFAULTS + [[ '' != \D\E\F\A\U\L\T\S ]] + echo '[DEFAULTS]' + section=DEFAULTS ++ echo DEFAULTS__minimum_diskGB ++ sed 's/.*__\(.*\)$/\1/' + local var_param=minimum_diskGB + echo 'minimum_diskGB = 4' + cat /etc/contrail/contrail-config-database-nodemgr.conf [DEFAULTS] http_server_ip=0.0.0.0 log_file=/var/log/contrail/config-database-nodemgr/contrail-config-database-nodemgr.log log_level=SYS_DEBUG log_local=1 hostip=10.0.0.50 db_port=9041 db_jmx_port=7201 db_use_ssl=False [COLLECTOR] server_list=10.0.0.50:8086 [SANDESH] introspect_ssl_enable=False sandesh_ssl_enable=False [DEFAULTS] minimum_diskGB = 4 + upgrade_old_logs contrail-config-database-nodemgr + local template=contrail-config-database-nodemgr ++ dirname /var/log/contrail/config-database-nodemgr + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/config-database-nodemgr + chmod 755 /var/log/contrail/config-database-nodemgr + mv -n '/var/log/contrail/*contrail-config-database-nodemgr.log*' /var/log/contrail/config-database-nodemgr/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/config-database-nodemgr + find /var/log/contrail/config-database-nodemgr -uid 0 -exec chown 1999:1999 '{}' + + exec /bin/sh -c '/usr/local/bin/contrail-nodemgr --nodetype=${NODEMGR_TYPE}' 08/28/2025 17:19:55.944 7efe74741040 [contrail-config-database-nodemgr] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 08/28/2025 17:19:56.342 7efe74741040 [contrail-config-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 17:19:56.360 7efe74741040 [contrail-config-database-nodemgr] [INFO]: SANDESH: INTROSPECT IS ON: 0.0.0.0:8112 08/28/2025 17:19:56.376 7efe74741040 [contrail-config-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 17:19:56.383 7efe74741040 [contrail-config-database-nodemgr] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 08/28/2025 17:19:56.396 7efe74741040 [contrail-config-database-nodemgr] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/config-database-nodemgr/contrail-config-database-nodemgr.log] Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: CRITICAL:root:Exit data: {'ID': '164dec4bcbc2fa86366633346c487573f2e5791e2cb2a913cd9e64fbadc1b20f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 230239} ERROR:contrail-config-database-nodemgr:Failed to connect to database by CQL and socket: Result: 1 INFO:contrail-config-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-config-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1437-1', _node_type='ConfigDatabase', _instance_id='0', _timestamp=1756401598341667, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-config-database-nodemgr', instance_id='0', state='Non-Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=[''], status='Down', description='none to Idle on EvStart')]), flag_infos=None, description='Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo'), _seqnum=0) INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:19:58] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 4555 0.002100 INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8112 start_time = 1756401596410179 collector_name = collector_ip = 10.0.0.50:8086 collector_list = [ 10.0.0.50:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': 'b0f08baad15884bf8c58d8beb1c9c1829d01799e83f11982ccbb896973cac2fa', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 230284} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 CRITICAL:root:Exit data: {'ID': '6306173ffdd1a27b5224a52eed64ec5c4e67845a37d1bfa37c7ea4fcf96ec7ec', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'config_db_uuid'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 230750} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr config_db_uuid'. Error is: Result: 1 INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8112 start_time = 1756401596410179 collector_name = collector_ip = 10.0.0.50:8086 collector_list = [ 10.0.0.50:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': 'f646330535b79b4543fb5a5ca60d7e0d4319e985368208ae580e365a56dac19e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'to_bgp_keyspace'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 231255} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr to_bgp_keyspace'. Error is: Result: 1 CRITICAL:root:Exit data: {'ID': 'da3665e3111d4a97f9510cd2fe0ae1d01301d58a1b362e32e913f9b29b5eeb7f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'svc_monitor_keyspace'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 232126} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr svc_monitor_keyspace'. Error is: Result: 1 CRITICAL:root:Exit data: {'ID': 'c9298837bbdb88af68221afe91a160b0b376f4ce9c731c849f6f8f168a93d85e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'DISCOVERY_SERVER'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 233470} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr DISCOVERY_SERVER'. Error is: Result: 1 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:20:08] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 4593 0.001716 INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8112 start_time = 1756401596410179 collector_name = collector_ip = 10.0.0.50:8086 collector_list = [ 10.0.0.50:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': 'aa0f510952545605e2ac986b23f7d2481f8e3263c6bacf55e8121059b1a36ab5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'useragent'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 233561} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr useragent'. Error is: Result: 1 CRITICAL:root:Exit data: {'ID': '8137a3794e650cd4f411b01bb2a7471fe75d98c9e832b870fe7549463786e476', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'dm_keyspace'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 233743} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr dm_keyspace'. Error is: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8112 start_time = 1756401596410179 collector_name = collector_ip = 10.0.0.50:8086 collector_list = [ 10.0.0.50:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:send_process_state_db: Sending UVE: NodeStatusUVE(_context='', _scope='', _module='contrail-config-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1437-1', _node_type='ConfigDatabase', _instance_id='0', _timestamp=1756401614601398, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=False, status=None, process_status=None, process_info=[ProcessInfo(process_name='cassandra', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756401591000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='zookeeper', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756401591000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-config-database-nodemgr', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756401591000000', last_stop_time='', last_exit_time='', core_file_list=[])], disk_usage_info=None, description=None, all_core_file_list=None, build_info='{"build-info" : [{"build-version" : "v0+master.2.0.4.1.1", "build-time" : "2025-08-28 16:00:30.009979", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10619-1.novalocal", "build-id" : "v0+master.2.0.4.1.1", "build-number" : "master_2_0_4_1_1.el9"}]}', process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo'), _seqnum=0) DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=11533268, partition_space_available_1k=69557216, percentage_partition_space_used=14, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=[], build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1271500, cpu_share=0.0, mem_res=1259324, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56992, cpu_share=0.0, mem_res=52568, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54244, cpu_share=0.0, mem_res=42776, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=4438244, free=206372, buffers=448332, cached=11091256, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.02, five_min_avg=2.19, fifteen_min_avg=1.03, cpu_share=0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 deleted = False process_info = [ << process_name = cassandra process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756401591000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = zookeeper process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756401591000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-config-database-nodemgr process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756401591000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, ] build_info = {"build-info" : [{"build-version" : "v0+master.2.0.4.1.1", "build-time" : "2025-08-28 16:00:30.009979", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10619-1.novalocal", "build-id" : "v0+master.2.0.4.1.1", "build-number" : "master_2_0_4_1_1.el9"}]} >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 disk_usage_info = { /dev/vda1 : << partition_type = ext4 partition_space_used_1k = 11533268 partition_space_available_1k = 69557216 percentage_partition_space_used = 14 >> } all_core_file_list = [ ] process_mem_cpu_usage = { cassandra : << mem_virt = 1271500 cpu_share = 0.0 mem_res = 1259324 >>zookeeper : << mem_virt = 56992 cpu_share = 0.0 mem_res = 52568 >>contrail-config-database-nodemgr : << mem_virt = 54244 cpu_share = 0.0 mem_res = 42776 >> } system_mem_usage = << total = 16184204 used = 4438244 free = 206372 buffers = 448332 cached = 11091256 node_type = config-database-node >> system_cpu_usage = << one_min_avg = 5.02 five_min_avg = 2.19 fifteen_min_avg = 1.03 cpu_share = 0 node_type = config-database-node >> >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8112 start_time = 1756401596410179 collector_name = collector_ip = 10.0.0.50:8086 collector_list = [ 10.0.0.50:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:20:18] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6922 0.001555 INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8112 start_time = 1756401596410179 collector_name = collector_ip = 10.0.0.50:8086 collector_list = [ 10.0.0.50:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-config-database-nodemgr:Session Event: TCP Connected INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1437-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-config-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] DEBUG:contrail-config-database-nodemgr:Extracted sandesh length: 0000001013 DEBUG:contrail-config-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[ClientInit] DEBUG:contrail-config-database-nodemgr:Number of uve types in sandesh control message is 0 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:DatabaseUsage] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvSandeshCtrlMessageRecv] => State[ClientInit] -> State[Established] INFO:contrail-config-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-config-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:20:52] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6866 0.002048 DEBUG:contrail-config-database-nodemgr:Extracted sandesh length: 0000001434 DEBUG:contrail-config-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[Established] DEBUG:contrail-config-database-nodemgr:Number of uve types in sandesh control message is 3 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:DatabaseUsage] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-config-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-config-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '50c4207671d3bfd37aa9436b50fb7c461aeed6614d9ecd2030259763e0f65a36', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 240808} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open INFO:contrail-config-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-config-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1437-1', _node_type='ConfigDatabase', _instance_id='0', _timestamp=1756401660420863, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-config-database-nodemgr', instance_id='0', state='Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=['10.0.0.50:8086'], status='Up', description='ClientInit to Established on EvSandeshCtrlMessageRecv')]), flag_infos=None, description='')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo'), _seqnum=0) CRITICAL:root:Exit data: {'ID': 'a335e66282afdc4bb1e50745c4daf0b06ea23d3252c5950be76b89855c1406dd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 240857} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12411256, partition_space_available_1k=68679228, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1341964, cpu_share=0.39, mem_res=1325860, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=177856, cpu_share=0.13, mem_res=172428, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53456, cpu_share=0.01, mem_res=42536, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=6921224, free=235820, buffers=404248, cached=8622912, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=13.62, five_min_avg=5.02, fifteen_min_avg=2.06, cpu_share=22.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '407fce94a62a8874b6a30ccc2ea3ee4d3dccf081f5e5e6bec753ae4022417bd3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 250138} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a81f8c69dee52c4c18a10a05567ec3b5c405104ad7a3a4b97ff90c7c95be7eef', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 250191} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12441264, partition_space_available_1k=68649220, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1344940, cpu_share=0.33, mem_res=1328228, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=230788, cpu_share=0.04, mem_res=225228, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53428, cpu_share=0.01, mem_res=42692, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7374864, free=214912, buffers=428720, cached=8165708, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=14.28, five_min_avg=6.87, fifteen_min_avg=2.87, cpu_share=20.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0653e953b47893c4a7c76997882b7139270f03e65c29d78a2f3e21eb111877ee', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 263448} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3c4cffa673140434cd2f73f486dd697cbe28c39dda36beffed40dab5eecaa879', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 263506} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:23:00] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6840 0.001811 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12463588, partition_space_available_1k=68626896, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1343604, cpu_share=0.04, mem_res=1327448, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=230788, cpu_share=0.0, mem_res=225228, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53564, cpu_share=0.01, mem_res=42840, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7371968, free=455012, buffers=423900, cached=7933324, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=8.28, five_min_avg=6.5, fifteen_min_avg=2.99, cpu_share=13.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:23:11] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001764 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:23:15] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001760 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:23:44] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.002157 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:23:51] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001594 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bafbc1e0c3d244aa56f9f891155ccc735501fb9f26db7460db378a918f4b7aec', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 277112} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:23:57] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001475 CRITICAL:root:Exit data: {'ID': 'c48e9ac8afcd372f96e5ce6cc3fb5414d1b96b4de4d25df443dab33c47207d00', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 277134} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12479332, partition_space_available_1k=68611152, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1348260, cpu_share=0.06, mem_res=1331728, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=230804, cpu_share=0.0, mem_res=225228, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53476, cpu_share=0.01, mem_res=42656, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7708932, free=168996, buffers=431312, cached=7874964, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=6.19, five_min_avg=6.17, fifteen_min_avg=3.1, cpu_share=15.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:24:06] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.002069 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:24:13] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.002123 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:24:18] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001383 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:24:37] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001473 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '19d59ea798a128d6055286dada2fbf5ac1e29345ee6a1b1dd45718486c714cd2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 281841} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b9ad8c49281b86b654e207c8a3e9632c2a72941bea3fc87e7654d18c51478811', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 281865} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12499608, partition_space_available_1k=68590876, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1351336, cpu_share=0.21, mem_res=1334672, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=230876, cpu_share=0.01, mem_res=225228, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53556, cpu_share=0.01, mem_res=42676, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7644984, free=296516, buffers=428856, cached=7813848, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.38, five_min_avg=6.24, fifteen_min_avg=3.33, cpu_share=12.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e6e98261bb66eda291ca49fa3636848fdb7e035b1a5896de049de03ed5f63a6e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 282951} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '69eb3456e68235705b1488f063353fc3869c1aba2fdfe33fa232c56d855fa110', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 282976} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12506188, partition_space_available_1k=68584296, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1352412, cpu_share=0.04, mem_res=1335208, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=230880, cpu_share=0.0, mem_res=225228, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53704, cpu_share=0.01, mem_res=42832, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7655076, free=277876, buffers=428888, cached=7822364, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.16, five_min_avg=5.51, fifteen_min_avg=3.26, cpu_share=8.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:26:54] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001717 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9d378f92d20d46b79d566094180bd762db0d7d3d6071bf2c785d2430e971097a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 286174} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:26:58] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001441 CRITICAL:root:Exit data: {'ID': '3322e569878b74442b5a1e0090c95e2b2c764a913bd7b8843ffb581210032b0c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 286196} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:27:03] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.002452 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12515348, partition_space_available_1k=68575136, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1352272, cpu_share=0.04, mem_res=1335276, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=230892, cpu_share=0.0, mem_res=225360, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53700, cpu_share=0.01, mem_res=42752, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7680200, free=264732, buffers=428656, cached=7810616, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.33, five_min_avg=5.09, fifteen_min_avg=3.26, cpu_share=9.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '40a76c595888b6d61d8703f2e441e4d5470b8d9ec68f3117c58e6552d9bf05e8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 291237} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6b9f827f26db16763a1fc9146cff672e00cbcc3ab18b63b98a0eca7ecddcefd4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 291268} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12525312, partition_space_available_1k=68565172, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1352364, cpu_share=0.05, mem_res=1335168, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231192, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53620, cpu_share=0.01, mem_res=42744, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7695792, free=233516, buffers=429616, cached=7825280, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.57, five_min_avg=4.52, fifteen_min_avg=3.17, cpu_share=10.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9582b1b79fe9938d495ccdb79772468a70f64f460ea72616492171674ad02ae8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 295391} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ce1a7ef9cc2d870948fe6a34355cb89a6e1ec5dc2a8da1c093156c320f587499', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 295415} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12541592, partition_space_available_1k=68548892, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1352300, cpu_share=0.04, mem_res=1335024, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231192, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53776, cpu_share=0.01, mem_res=42760, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7697908, free=294468, buffers=429252, cached=7762576, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.98, five_min_avg=4.01, fifteen_min_avg=3.08, cpu_share=9.25, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:29:17] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001477 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c3f16dd00ba36a0b305a1ef8c44aebde8bb024156cfa8e10672e245172c02ea7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 297111} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7d780760f0b020ce1a00559147674a81960ce4fe5098c3ace0403b7a2a4b2dae', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 297132} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12549660, partition_space_available_1k=68540824, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1352536, cpu_share=0.03, mem_res=1335484, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231196, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53776, cpu_share=0.01, mem_res=42752, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7720296, free=258856, buffers=430156, cached=7774896, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.66, five_min_avg=3.54, fifteen_min_avg=2.97, cpu_share=8.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '41dcf914ab7bf6c705a609a1724bff8ee8882ca1b54d7ea16d0b92bd89dc39de', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 298214} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c000c28b95e640de68db28a84c6d9574e4a8db47a7b0a331602f34d997a13bd7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 298237} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12557092, partition_space_available_1k=68533392, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1353812, cpu_share=0.04, mem_res=1336092, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231196, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53660, cpu_share=0.01, mem_res=42652, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7670372, free=299532, buffers=430204, cached=7784096, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.58, five_min_avg=3.18, fifteen_min_avg=2.89, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:31:44] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.002055 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:31:47] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001299 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:31:51] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001265 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a29435a284cb6764e6d58dc7f12b680fdd440c5090df0d411ff267f6ceb0da8c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 303721} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b5b214c5b7abe132e76d6dab60d7f848e6bfb14680e004573fe023f9921c8bfb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 303746} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12563276, partition_space_available_1k=68527208, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1353784, cpu_share=0.04, mem_res=1336028, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231200, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53816, cpu_share=0.01, mem_res=42760, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7678872, free=282856, buffers=430264, cached=7792212, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.68, five_min_avg=2.9, fifteen_min_avg=2.81, cpu_share=8.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b1078c275b9d72481d83b355e9673dc980374f1c60fed2a9dc26950bc1b4d0aa', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 305156} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7cb178f8fbb37aa26e4598ba8e682af4ced816dbdce8c2cf232fd3492ea7bbc7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 305177} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12565756, partition_space_available_1k=68524728, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1353860, cpu_share=0.03, mem_res=1335948, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231200, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53976, cpu_share=0.01, mem_res=42808, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7704084, free=252988, buffers=430600, cached=7796532, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.5, five_min_avg=2.64, fifteen_min_avg=2.72, cpu_share=7.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:33:35] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001645 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e57b8a6b16d1366427d38bc7675922771258fc2b6f34919ebb4b40451274a354', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 311206} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8b059f4dcbc63c591fb2c079de9febb2448063a3b61f19618110840d7282dfae', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 311229} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12569432, partition_space_available_1k=68521052, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1353780, cpu_share=0.03, mem_res=1335988, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231200, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54008, cpu_share=0.01, mem_res=42732, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7695384, free=506712, buffers=430964, cached=7551144, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.62, five_min_avg=2.48, fifteen_min_avg=2.66, cpu_share=10.25, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '13ef2ed80f9b591ea086add7b69b79c3bad77c7d4b60b52e1568e560fb003126', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 312303} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b1dd66ae95d0b0ad6c9a49df89374d8ecfa32dfcc1abe06698a69739badaddef', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 312327} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12571868, partition_space_available_1k=68518616, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1353744, cpu_share=0.04, mem_res=1335824, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231200, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53952, cpu_share=0.01, mem_res=42680, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7704420, free=492768, buffers=431036, cached=7555980, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.0, five_min_avg=2.43, fifteen_min_avg=2.63, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b6bac1eb98a970c0bc1923c6466e14fcf0729c060f9f2251a41b3557ac6cf313', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 313728} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '074e26c01e18f6842a5ba66c58639431813ec7739c32a44f45b7ef55f70ac711', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 313842} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:36:00] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6835 0.001746 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:36:03] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6835 0.001792 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12575776, partition_space_available_1k=68514708, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355096, cpu_share=0.04, mem_res=1336852, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231200, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53976, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7722528, free=468188, buffers=431104, cached=7562384, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.7, five_min_avg=2.54, fifteen_min_avg=2.66, cpu_share=8.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:36:07] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6835 0.001510 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b1f38f96fd7a2274a36cc66256f7e025ad1417b68f5b021c1d7d70fef8847721', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 318909} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'df55709c598665cfd4a9383485d6e3d012f6f5671c546ce5a45d8a09351c4599', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 318930} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12585528, partition_space_available_1k=68504956, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355104, cpu_share=0.04, mem_res=1336756, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231200, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54128, cpu_share=0.01, mem_res=42924, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7686984, free=493812, buffers=431148, cached=7572260, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.44, five_min_avg=2.51, fifteen_min_avg=2.64, cpu_share=8.25, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:37:43] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001454 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4f50bb613da8128e029c2200414a538ffe0699606e67c1eff54010dfdab79250', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 320966} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '50a6107e399a87861662161df5eab032c752b140e7b324b40240e3a2dbc29178', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 320991} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12587928, partition_space_available_1k=68502556, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355292, cpu_share=0.03, mem_res=1336912, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231200, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54192, cpu_share=0.01, mem_res=42784, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7696020, free=480456, buffers=431212, cached=7576516, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.02, five_min_avg=2.4, fifteen_min_avg=2.59, cpu_share=8.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4504b6416d100f99f36fe8dd3d46198b953f4a64a4b02d1669372f41ba5136ff', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 322065} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0e50c73abee4e9b422c21acf3f9ac5ed96af0dc1d46c3a89bfc9fee97c9d4686', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 322090} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12589948, partition_space_available_1k=68500536, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355148, cpu_share=0.03, mem_res=1336732, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54196, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7693188, free=480208, buffers=431224, cached=7579584, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.44, five_min_avg=2.38, fifteen_min_avg=2.57, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e3be245422013c886de2bb5ab771b0dfea8486769687771535789de37f5a811e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 327483} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2e9a173432098d42beb8c6e22c245d8803263f20738bc66d89f147b9b160143f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 327506} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12592792, partition_space_available_1k=68497692, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355260, cpu_share=0.03, mem_res=1336812, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54272, cpu_share=0.01, mem_res=42644, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7704100, free=460908, buffers=431312, cached=7587884, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.13, five_min_avg=2.32, fifteen_min_avg=2.54, cpu_share=10.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:40:15] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.002033 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:40:17] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001398 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:40:22] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001410 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '60792a688ffc2ac6014345d7fb56e9e532d368f2ed6966d9790ec1547fb8af52', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 333018} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1373f41e279008f26f549cc86c8727f5ea90ae4fb951ab57de8a1f5373745ba3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 333040} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12598900, partition_space_available_1k=68491584, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1356540, cpu_share=0.04, mem_res=1337764, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54228, cpu_share=0.01, mem_res=42668, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7708472, free=446880, buffers=431456, cached=7597396, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.01, five_min_avg=2.28, fifteen_min_avg=2.51, cpu_share=9.25, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6c2cd54989fc93bd199dac1b266ceb0ea77716ecbebd2aafd416dc5597ee5b86', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 334465} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '14f68b3213331a6cb174fb446d073b5ed20c5d170c3c0dd187a27384ec5b057a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 334488} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12601164, partition_space_available_1k=68489320, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1356596, cpu_share=0.03, mem_res=1337704, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54212, cpu_share=0.01, mem_res=42856, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7707236, free=444488, buffers=431468, cached=7601012, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.83, five_min_avg=2.19, fifteen_min_avg=2.46, cpu_share=8.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '257aa46c1b223e94b59ed2670827252bdbbeda51b1b10d2a08ed7a2a0f4874ff', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 335546} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1402465da247fd05ac2059fe580549ebcde7284e033709a4b9210e747c4f9533', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 335569} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12607968, partition_space_available_1k=68482516, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1356632, cpu_share=0.03, mem_res=1337564, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54108, cpu_share=0.01, mem_res=42808, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7711564, free=431968, buffers=431488, cached=7609184, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.76, five_min_avg=2.1, fifteen_min_avg=2.42, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2282fbe555353f1724b522acaefd09ba3c76e93c624d31659397dc46c9a88e6d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 341532} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:43:58] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6835 0.001462 CRITICAL:root:Exit data: {'ID': '60e1721ad428b675f9da8a8ea90311830977664749ee9bff907afb3e5973e672', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 341554} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12624368, partition_space_available_1k=68466116, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1356424, cpu_share=0.03, mem_res=1337508, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54244, cpu_share=0.01, mem_res=43040, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7724108, free=411692, buffers=431540, cached=7616864, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.9, five_min_avg=2.06, fifteen_min_avg=2.38, cpu_share=10.25, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:44:28] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001560 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:44:31] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001293 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:44:36] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001301 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '67762feca8ffdf53234b6eafa242bb43a6854b36d9356925e4d7bc42c38afec8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 347089} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3a783922d64c57506c9ffb860fc8cb7f7cd2a7d348f04551c59b717061b66686', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 347112} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12630492, partition_space_available_1k=68459992, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1356612, cpu_share=0.03, mem_res=1337836, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54420, cpu_share=0.01, mem_res=42720, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7732088, free=390684, buffers=431580, cached=7629852, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.06, five_min_avg=2.04, fifteen_min_avg=2.35, cpu_share=8.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a6fdf264609f844716e4d2b34713e9788383454329502015997222256ad87819', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 348550} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '90a3a1fa22b19b099fee59a4946718329ef019ebe72c8ed44e97136823af9832', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 348574} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12623308, partition_space_available_1k=68467176, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1357964, cpu_share=0.03, mem_res=1338764, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54424, cpu_share=0.01, mem_res=42764, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7737368, free=391040, buffers=431588, cached=7624208, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.65, five_min_avg=1.93, fifteen_min_avg=2.29, cpu_share=8.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e94889a66edbf4209d90cfc994e910f392f593945e7c3f51039a017f6d209465', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 349653} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '4289db8f5dbf9b962abfd0ec2f1db21232d4e06127e2486ac973f934359cc755', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 349676} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12625500, partition_space_available_1k=68464984, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1358080, cpu_share=0.03, mem_res=1338456, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54496, cpu_share=0.01, mem_res=42932, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7752060, free=373412, buffers=431600, cached=7627132, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.79, five_min_avg=1.88, fifteen_min_avg=2.25, cpu_share=7.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ef5b132e6f9dd483e4177e9a2eb847fa9c4aee59bbdd61e6f9ad66b145ac526e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 350738} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'eef738a16d44109c2bce5c564eef1e1af738f4f38d2c09fbf87f7be72a45730b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 350761} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12632564, partition_space_available_1k=68457920, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1357760, cpu_share=0.03, mem_res=1338624, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54540, cpu_share=0.01, mem_res=42848, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7758848, free=363308, buffers=431608, cached=7630440, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.56, five_min_avg=1.8, fifteen_min_avg=2.19, cpu_share=7.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:48:42] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001603 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '79af8189085aa559763b887f9389d3259df3e81186fa520c91af4dc25bb762a2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 356786} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a3a284f9852a576b9e3f63b051ac75a8b375222b79acb88dc5c758649741b372', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 356809} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12636248, partition_space_available_1k=68454236, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1357896, cpu_share=0.03, mem_res=1338560, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54524, cpu_share=0.01, mem_res=42884, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7759264, free=349872, buffers=431640, cached=7643428, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.36, five_min_avg=2.04, fifteen_min_avg=2.25, cpu_share=10.25, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:49:30] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6838 0.002042 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:49:33] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6838 0.001199 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:49:37] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6838 0.001350 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6d740805fc992a0a81183ff6a3d1984ab9f2ad6fc387b2ffb378f2d75211fd25', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 362635} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'eb2af8e2d031c0bc4de4e902e0b7057aad07512544c0b277951e365cbfbfbfc1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 362658} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12642264, partition_space_available_1k=68448220, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1358024, cpu_share=0.03, mem_res=1338572, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231204, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54404, cpu_share=0.01, mem_res=42828, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7781240, free=319312, buffers=431688, cached=7651964, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.29, five_min_avg=2.06, fifteen_min_avg=2.25, cpu_share=9.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c742d19243aa82c70416d6c15629282fb26d3da8f0e0a3f036e62d16ff25cb49', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 363733} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd217d2ca51b91838f6606e874b7d127b1ef1ba089e4411f405d28ad39cf82c3c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 363759} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12644648, partition_space_available_1k=68445836, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1359376, cpu_share=0.03, mem_res=1339316, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54428, cpu_share=0.01, mem_res=42880, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7796764, free=299888, buffers=431700, cached=7655852, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.98, five_min_avg=2.01, fifteen_min_avg=2.21, cpu_share=7.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1ff37ea40e06b4e5248fbbe1423543c148a8ff9008348ce76f21f1fc5575addc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 364807} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '005873e424a6dd5b8bfb55328cbe668e7c34c75492ba3250c80eee3e9e563bdd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 364831} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12646812, partition_space_available_1k=68443672, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1359328, cpu_share=0.03, mem_res=1339320, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54448, cpu_share=0.01, mem_res=42940, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7796016, free=297768, buffers=431720, cached=7658700, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.54, five_min_avg=2.11, fifteen_min_avg=2.23, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '701be09e9524c0d655e65e35b7232482f6bd94737bf310d5b0d080186fc77f4b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 365896} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c93f70d77c88af684f6a298b956e60c5d6353258cf85fd7fcd7739a18f4d2402', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 365919} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12649192, partition_space_available_1k=68441292, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1359272, cpu_share=0.03, mem_res=1339308, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54496, cpu_share=0.01, mem_res=42936, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7794384, free=295028, buffers=431752, cached=7663040, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.81, five_min_avg=1.97, fifteen_min_avg=2.17, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:53:22] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6836 0.001583 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8a8b9d7aef8548081d560ae205f5afcc858beb6db67c7ed339f7aafdd330b6e8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 371942} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5835caa364c2376be3e368158a9b75446a7d59bfeedb32a88e61adee0583b023', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 371965} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12656860, partition_space_available_1k=68433624, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1359164, cpu_share=0.03, mem_res=1339140, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54424, cpu_share=0.01, mem_res=42952, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7798288, free=283408, buffers=431772, cached=7670736, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.24, five_min_avg=2.03, fifteen_min_avg=2.18, cpu_share=10.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:54:17] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001457 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:54:20] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001471 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:54:25] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001590 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '34721c75831de8f2b47f2e9ff3b1f615414ecb1ffc734e83b6278cdfea242f4d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 377789} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '42530c2be042ce9e66f514b0490c3134b8f3b4d528fa81c3cfcf9ea8681505df', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 377813} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12662612, partition_space_available_1k=68427872, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1359300, cpu_share=0.03, mem_res=1339416, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54336, cpu_share=0.01, mem_res=42628, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7807464, free=261156, buffers=431800, cached=7683784, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.12, five_min_avg=2.39, fifteen_min_avg=2.3, cpu_share=9.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '18c32f9741a48424cb83ed8d084dd9ea2371717d7344ca53f501f8887c38fdc3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 378902} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '01c10ad6aa1b7cdd12e4a0ac6760ffc44fb4e7fceae7bb13ca717358d2ecf582', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 378926} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12664796, partition_space_available_1k=68425688, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1360564, cpu_share=0.03, mem_res=1340016, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54328, cpu_share=0.01, mem_res=42700, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7817104, free=247760, buffers=431840, cached=7687500, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.15, five_min_avg=2.21, fifteen_min_avg=2.24, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '3d9ef2e74a3c29bdc8ee55326dd86d49f424eaf87b7d6b1de5231e77c2369447', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 379976} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '07eec0e6a7a0f8cf93c2ce6b14e8d0910cb3c90b2d0ca71e551f0a9f5bb136d4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 379999} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12667188, partition_space_available_1k=68423296, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1360364, cpu_share=0.03, mem_res=1340040, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54464, cpu_share=0.01, mem_res=42848, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7815356, free=246340, buffers=431876, cached=7690632, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.86, five_min_avg=2.08, fifteen_min_avg=2.19, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4634b58c36f31b90de447e0bbec61f8b91887e5b47ad5b8e29c53753f8242344', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 381069} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6769f2fc2d65ddce3ca59b6fc9372aab095393c7950cb3cb628485c6fe20a2d4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 381091} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12673976, partition_space_available_1k=68416508, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1360576, cpu_share=0.03, mem_res=1339964, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54404, cpu_share=0.01, mem_res=42760, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7815052, free=238704, buffers=431892, cached=7698556, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.73, five_min_avg=1.97, fifteen_min_avg=2.14, cpu_share=8.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e62ed4ad69ce53f1b837c35c39a577dc847dd5b892fee0c69c609894a29ed86b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 386496} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b3a8529dfed992885f92068a2e9a4c7ca1d0b26d251ebad2a792fb32d3c81ac8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 386519} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12676636, partition_space_available_1k=68413848, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1360632, cpu_share=0.03, mem_res=1340020, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54652, cpu_share=0.01, mem_res=42756, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7836520, free=271820, buffers=430280, cached=7645584, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.5, five_min_avg=2.15, fifteen_min_avg=2.19, cpu_share=10.25, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:59:17] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001634 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:59:55] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001282 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'aca55179197b419c9184115f4dd010caa837e6d8cf321a7439c56da25b113168', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 390665} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 17:59:57] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001415 CRITICAL:root:Exit data: {'ID': 'a9bf0934f7f6b511de170391ffeeebba80fafc255f860d84afc2ac2bebb39038', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 390696} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 18:00:02] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001876 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12676180, partition_space_available_1k=68414304, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1360512, cpu_share=0.03, mem_res=1340192, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54672, cpu_share=0.01, mem_res=43076, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7866912, free=256240, buffers=429416, cached=7631636, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.62, five_min_avg=2.22, fifteen_min_avg=2.21, cpu_share=9.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6c7340e9783e291905c3612280f3b2bbe3d792810a003c8a57ea5cd3d17bfb4e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 394094} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '705c1a7b56e441a8af1e53f73b9a026096f96bf5d68a20b3771b6774a11c126b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 394115} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12680608, partition_space_available_1k=68409876, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1361796, cpu_share=0.03, mem_res=1340876, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54540, cpu_share=0.01, mem_res=42948, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7847996, free=261052, buffers=430284, cached=7644872, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.96, five_min_avg=2.1, fifteen_min_avg=2.17, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '808f0ad5369228d1bf16dee56c8e340b36a78a6fbcfd01366e8935955704d527', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 395195} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7df767fda79747e276787144ea5b0cacf6de7e37b57ad86fe114c13f1e46c6c1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 395219} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12683208, partition_space_available_1k=68407276, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1361924, cpu_share=0.03, mem_res=1340796, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54568, cpu_share=0.01, mem_res=42956, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7779956, free=324572, buffers=430308, cached=7649368, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.49, five_min_avg=2.23, fifteen_min_avg=2.21, cpu_share=7.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1ba6794d3f023219766cc0975ec9c054fbc5bbc7088a53ab8068407ca8bcc297', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 396283} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6aba6c477fc2195179094429fb8e2088d71653792d27642c7dbd1d548eb21685', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 396305} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12685100, partition_space_available_1k=68405384, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1361904, cpu_share=0.03, mem_res=1340712, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54580, cpu_share=0.01, mem_res=42932, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7781512, free=319088, buffers=430476, cached=7653128, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.32, five_min_avg=2.25, fifteen_min_avg=2.22, cpu_share=7.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6c753b8d99583bc3def9dd4d955dbfb924fddc3057c92511ba144b0fb69d39df', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 397374} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a4e25573eaa5480dea34437507d8db84ad28e46914bd3123d407d46134cd4484', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 397409} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1437-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12686944, partition_space_available_1k=68403540, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1362024, cpu_share=0.03, mem_res=1340800, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=231244, cpu_share=0.0, mem_res=225624, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54456, cpu_share=0.01, mem_res=42996, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7780416, free=317500, buffers=430492, cached=7655796, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.55, five_min_avg=2.05, fifteen_min_avg=2.15, cpu_share=7.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.50 - - [2025-08-28 18:04:12] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6837 0.001498 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e81829cbf921b729ad7164862db7435baac516e48f5f6f51c0da9ad3cae9980c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 403862} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'eb9dff127464606e74f18cf296b614a810daa8706d7eea3dc9c29e5ee42da3a3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '13c531df0144885e5cd03a9119fd88330c9803fec4daa03b22e3bf5df0d3f369', 'DetachKeys': '', 'Pid': 403922} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: