++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ mkdir -p /var/log/contrail/database-nodemgr ++ log_file=/var/log/contrail/database-nodemgr/console.log ++ touch /var/log/contrail/database-nodemgr/console.log ++ chmod 600 /var/log/contrail/database-nodemgr/console.log ++ exec +++ tee -a /var/log/contrail/database-nodemgr/console.log +++ date ++ echo 'INFO: =================== Sat Aug 30 11:43:27 PM UTC 2025 ===================' INFO: =================== Sat Aug 30 11:43:27 PM UTC 2025 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ ip route get 1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ get_cidr_for_nic ens3 +++ cut -d / -f 1 +++ command -v ip +++ local nic=ens3 +++ grep 'inet ' +++ head -n 1 +++ awk '{print $2}' +++ ip addr show dev ens3 ++ DEFAULT_LOCAL_IP=10.0.0.49 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=0 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 0 ]] ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=kubernetes ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=no-auth ++ AUTH_MODE=noauth ++ AUTH_PARAMS= ++ SSL_ENABLE=False ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.49 ++ ANALYTICS_ALARM_ENABLE=True ++ ANALYTICS_SNMP_ENABLE=True ++ ANALYTICSDB_ENABLE=True ++ ANALYTICS_NODES=10.0.0.49 ++ ANALYTICSDB_NODES=10.0.0.49 ++ ANALYTICS_SNMP_NODES=10.0.0.49 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:8081 ' +++ '[' -n '10.0.0.49:8081 ' ']' +++ echo 10.0.0.49:8081 ++ ANALYTICS_SERVERS=10.0.0.49:8081 +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:9042 ' +++ '[' -n '10.0.0.49:9042 ' ']' +++ echo 10.0.0.49:9042 ++ ANALYTICSDB_CQL_SERVERS=10.0.0.49:9042 ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.49 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:8086 ' +++ '[' -n '10.0.0.49:8086 ' ']' +++ echo 10.0.0.49:8086 ++ COLLECTOR_SERVERS=10.0.0.49:8086 ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.49 ++ CONFIGDB_NODES=10.0.0.49 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:8082 ' +++ '[' -n '10.0.0.49:8082 ' ']' +++ echo 10.0.0.49:8082 ++ CONFIG_SERVERS=10.0.0.49:8082 +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:9161 ' +++ '[' -n '10.0.0.49:9161 ' ']' +++ echo 10.0.0.49:9161 ++ CONFIGDB_SERVERS=10.0.0.49:9161 +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:9041 ' +++ '[' -n '10.0.0.49:9041 ' ']' +++ echo 10.0.0.49:9041 ++ CONFIGDB_CQL_SERVERS=10.0.0.49:9041 ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=False ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=False ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=False ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.0.0.49 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.0.0.49 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.49 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+=10.0.0.49:2181, +++ '[' -n 10.0.0.49:2181, ']' +++ echo 10.0.0.49:2181 ++ ZOOKEEPER_SERVERS=10.0.0.49:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:2181 ' +++ '[' -n '10.0.0.49:2181 ' ']' +++ echo 10.0.0.49:2181 ++ ZOOKEEPER_SERVERS_SPACE_DELIM=10.0.0.49:2181 ++ RABBITMQ_NODES=10.0.0.49 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+=10.0.0.49:5673, +++ '[' -n 10.0.0.49:5673, ']' +++ echo 10.0.0.49:5673 ++ RABBITMQ_SERVERS=10.0.0.49:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=False ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:6379 ' +++ '[' -n '10.0.0.49:6379 ' ']' +++ echo 10.0.0.49:6379 ++ REDIS_SERVERS=10.0.0.49:6379 ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=False ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.49 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.49 +++ local server_address=10.0.0.49 +++ extended_server_list+='10.0.0.49:9092 ' +++ '[' -n '10.0.0.49:9092 ' ']' +++ echo 10.0.0.49:9092 ++ KAFKA_SERVERS=10.0.0.49:9092 ++ KAFKA_SSL_ENABLE=False ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/ ++ KEYSTONE_AUTH_HOST=127.0.0.1 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.49 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.49 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=False ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=False ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=False ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ / == \/\v\2\.\0 ]] ++ [[ kubernetes == \o\p\e\n\s\t\a\c\k ]] ++ [[ noauth == \k\e\y\s\t\o\n\e ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ noauth == \k\e\y\s\t\o\n\e ]] + cat + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + [[ database == \t\o\r\a\g\e\n\t ]] + export NODEMGR_TYPE=contrail-database + NODEMGR_TYPE=contrail-database + NODEMGR_NAME=contrail-database-nodemgr ++ echo DATABASE ++ tr - _ + ntype=DATABASE + [[ DATABASE == \V\R\O\U\T\E\R ]] + [[ DATABASE == \D\A\T\A\B\A\S\E ]] + htype=ANALYTICSDB + [[ DATABASE == \C\O\N\T\R\O\L ]] ++ get_listen_ip_for_node ANALYTICSDB +++ find_my_ip_and_order_for_node ANALYTICSDB +++ local server_typ=ANALYTICSDB_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.49 +++ local servers=10.0.0.49 +++ local server_list= +++ IFS=, +++ read -ra server_list ++++ tr '\n' , ++++ get_local_ips ++++ sort ++++ grep -vi host ++++ awk '/32 host/ { print f } {f=$2}' ++++ uniq +++ cut -d ' ' -f 1 ++++ cat /proc/net/fib_trie +++ local local_ips=,10.0.0.49,10.245.192.1,127.0.0.1,172.17.0.1,252.49.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.49'\''))' +++ local server_ip=10.0.0.49 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.49 ]] +++ [[ ,10.0.0.49,10.245.192.1,127.0.0.1,172.17.0.1,252.49.0.1,, =~ ,10\.0\.0\.49, ]] +++ echo 10.0.0.49 1 +++ return ++ local ip=10.0.0.49 ++ [[ -z 10.0.0.49 ]] ++ echo 10.0.0.49 + hostip=10.0.0.49 + introspect_ip=0.0.0.0 + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + mkdir -p /etc/contrail + cat ++ format_boolean False ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ echo False + '[' -n '' ']' + cat + add_ini_params_from_env DATABASE_NODEMGR /etc/contrail/contrail-database-nodemgr.conf + local service_name=DATABASE_NODEMGR + local cfg_path=/etc/contrail/contrail-database-nodemgr.conf + local delim=__ ++ set -o posix ++ set ++ sort ++ grep '^DATABASE_NODEMGR__.*__.*=.*$' ++ sed 's/^DATABASE_NODEMGR__//g' ++ cut -d = -f 1 + local vars=DEFAULTS__minimum_diskGB + local section= + for var in $vars + local var_name=DATABASE_NODEMGR__DEFAULTS__minimum_diskGB + local val=4 ++ echo DEFAULTS__minimum_diskGB ++ sed s/DASH/-/g + local var_unsubst=DEFAULTS__minimum_diskGB ++ echo DEFAULTS__minimum_diskGB ++ sed 's/^\(.*\)__.*$/\1/' + local var_section=DEFAULTS + [[ '' != \D\E\F\A\U\L\T\S ]] + echo '[DEFAULTS]' + section=DEFAULTS ++ sed 's/.*__\(.*\)$/\1/' ++ echo DEFAULTS__minimum_diskGB + local var_param=minimum_diskGB + echo 'minimum_diskGB = 4' + cat /etc/contrail/contrail-database-nodemgr.conf [DEFAULTS] http_server_ip=0.0.0.0 log_file=/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log log_level=SYS_DEBUG log_local=1 hostip=10.0.0.49 db_port=9042 db_jmx_port=7200 db_use_ssl=False [COLLECTOR] server_list=10.0.0.49:8086 [SANDESH] introspect_ssl_enable=False sandesh_ssl_enable=False [DEFAULTS] minimum_diskGB = 4 + upgrade_old_logs contrail-database-nodemgr + local template=contrail-database-nodemgr ++ dirname /var/log/contrail/database-nodemgr + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/database-nodemgr + chmod 755 /var/log/contrail/database-nodemgr + mv -n '/var/log/contrail/*contrail-database-nodemgr.log*' /var/log/contrail/database-nodemgr/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/database-nodemgr + find /var/log/contrail/database-nodemgr -uid 0 -exec chown 1999:1999 '{}' + + exec /bin/sh -c '/usr/local/bin/contrail-nodemgr --nodetype=${NODEMGR_TYPE}' 08/30/2025 23:43:28.746 7f134922a900 [contrail-database-nodemgr] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 08/30/2025 23:43:28.857 7f134922a900 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/30/2025 23:43:28.866 7f134922a900 [contrail-database-nodemgr] [INFO]: SANDESH: INTROSPECT IS ON: 0.0.0.0:8103 08/30/2025 23:43:28.869 7f134922a900 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/30/2025 23:43:28.875 7f134922a900 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 08/30/2025 23:43:28.876 7f134922a900 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log] Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: CRITICAL:root:Exit data: {'ID': 'cd2af7291306a1a917f15335bc9eae75e380ca268c55e8e5a44d2ce58d806a6c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 263838} ERROR:contrail-database-nodemgr:Failed to connect to database by CQL and socket: Result: 1 INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1450-1', _node_type='Database', _instance_id='0', _timestamp=1756597409724654, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Non-Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=[''], status='Down', description='none to Idle on EvStart')]), flag_infos=None, description='Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': '1e9441cc9106486e117d4f5e95df2b98e56669ffc817f519f946415dbf32ea4f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 263861} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 CRITICAL:root:Exit data: {'ID': '7b32396c86c801e13971344e1dc94f3af8359fc066b0393c42e4d150d078e6f6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'repair', '-pr', 'ContrailAnalyticsCql'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 263982} ERROR:contrail-database-nodemgr:Failed to run cmd: 'nodetool -p 7200 repair -pr ContrailAnalyticsCql'. Error is: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.49:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.49:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756597408891479 collector_name = collector_ip = 10.0.0.49:8086 collector_list = [ 10.0.0.49:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-database-nodemgr:Session Event: TCP Connected INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.49:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:send_process_state_db: Sending UVE: NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1450-1', _node_type='Database', _instance_id='0', _timestamp=1756597417265026, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=False, status=None, process_status=None, process_info=[ProcessInfo(process_name='cassandra', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756597406000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-query-engine', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756597406000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-database-nodemgr', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756597406000000', last_stop_time='', last_exit_time='', core_file_list=[])], disk_usage_info=None, description=None, all_core_file_list=None, build_info='{"build-info" : [{"build-version" : "v0+master.2.0.8.7.1", "build-time" : "2025-08-30 22:19:30.890684", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10703-1.novalocal", "build-id" : "v0+master.2.0.8.7.1", "build-number" : "master_2_0_8_7_1.el9"}]}', process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=11578288, partition_space_available_1k=69512196, percentage_partition_space_used=14, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=[], build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1264988, cpu_share=0.0, mem_res=1252876, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=9424, cpu_share=0.0, mem_res=5564, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54804, cpu_share=0.0, mem_res=42440, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=5073012, free=177104, buffers=426376, cached=10507712, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.42, five_min_avg=1.79, fifteen_min_avg=0.95, cpu_share=0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] ERROR:contrail-database-nodemgr:Session Event: TCP Connection Closed INFO:contrail-database-nodemgr:Processing event[EvTcpClose] in state[ClientInit] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpClose] => State[ClientInit] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.49:8086, ] status = Down description = ClientInit to Idle on EvTcpClose >>, ] description = Cassandra state detected DOWN. Collector:Collector[ClientInit to Idle on EvTcpClose] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 1 pid = 1 http_port = 8103 start_time = 1756597408891479 collector_name = collector_ip = 10.0.0.49:8086 collector_list = [ 10.0.0.49:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-database-nodemgr:Session Event: TCP Connected INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1450-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.49:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001013 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[ClientInit] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 0 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Client: Event[EvSandeshCtrlMessageRecv] => State[ClientInit] -> State[Established] INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:43:56] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6847 0.001361 DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001434 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[Established] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 3 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:44:10] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6847 0.001429 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:44:25] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6847 0.001242 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c8afdcda2b23bbbcdbf3cc16c29bb4dd71c14f24f922b4d315a05b6be2d51d9b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 273116} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1450-1', _node_type='Database', _instance_id='0', _timestamp=1756597470506916, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=['10.0.0.49:8086'], status='Up', description='ClientInit to Established on EvSandeshCtrlMessageRecv')]), flag_infos=None, description='')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:44:31] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6812 0.001186 CRITICAL:root:Exit data: {'ID': '8a0d8e372d4c9d35649c8dcf62f9f5c4df37661acb59678b3fca5095104cfa62', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 273137} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:44:35] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6812 0.002291 DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12564204, partition_space_available_1k=68526280, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1348272, cpu_share=0.36, mem_res=1326580, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12864, cpu_share=0.0, mem_res=8580, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53808, cpu_share=0.01, mem_res=42540, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7565364, free=207684, buffers=409536, cached=8001620, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=14.89, five_min_avg=5.16, fifteen_min_avg=2.15, cpu_share=22.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:44:41] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6818 0.001385 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:44:45] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6818 0.001543 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a22c7bb467cabc074f82f2acae4385095a2f45cc2646a2872a284a7d8b937b1c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 277021} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '918f175df784eea73d65b1a378bc3ad732c77cf0b28d7575a0aa5097f6e1ea72', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 277042} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12487368, partition_space_available_1k=68603116, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1354352, cpu_share=0.12, mem_res=1329068, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13168, cpu_share=0.0, mem_res=8976, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54072, cpu_share=0.01, mem_res=42588, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7523384, free=321148, buffers=411240, cached=7928432, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=8.0, five_min_avg=4.99, fifteen_min_avg=2.28, cpu_share=10.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:45:57] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001525 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:46:00] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001331 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:46:04] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001475 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '870dae0550d6314cc62d6b2aa30a73d5944e8962e0cafa490d8d8dfd1779e439', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 282856} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '60b7e1ec18ee78be93be79b9cd7ec267338de8a72bfd5f8921acf0798e73ac74', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 282876} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12497724, partition_space_available_1k=68592760, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1358496, cpu_share=0.05, mem_res=1331252, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13364, cpu_share=0.0, mem_res=9108, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54012, cpu_share=0.01, mem_res=42644, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7532028, free=298196, buffers=411596, cached=7942384, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.35, five_min_avg=4.47, fifteen_min_avg=2.27, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f28f5b030527052654e777f8f4d89282fc8d7134e36d276251d21235389af40e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 285831} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ba1f6c5920600fcea094e64173fb9fb6c64ae219627dfc063cd9a6ec40f1c675', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 285893} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12505756, partition_space_available_1k=68584728, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1360812, cpu_share=0.04, mem_res=1331976, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13436, cpu_share=0.0, mem_res=9240, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54044, cpu_share=0.01, mem_res=42504, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7640748, free=226992, buffers=427752, cached=7888712, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.49, five_min_avg=4.12, fifteen_min_avg=2.29, cpu_share=9.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '83321c17106b0389bae4e4b5451aa00b5dc55fb23716596bdfc8f129edfd50c3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 289444} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '24f6bce60c14b949275984789f5044ff28721b683ac1adc8ac2c9f3b19650fcc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 289464} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12512836, partition_space_available_1k=68577648, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1363396, cpu_share=0.04, mem_res=1332584, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13488, cpu_share=0.0, mem_res=9240, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54224, cpu_share=0.01, mem_res=42532, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7588900, free=261004, buffers=427976, cached=7906324, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.2, five_min_avg=3.64, fifteen_min_avg=2.24, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1f52f47899fd7ef9e5aa2f68719f6cfe87738846ebfe1438ffb4010f19830094', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 290894} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '30ae95cb37b8755ecbee817801b52119f4c6781b1993d29e8f9c372721de3936', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 291012} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:49:31] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001985 DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12523876, partition_space_available_1k=68566608, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1365568, cpu_share=0.04, mem_res=1333468, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13580, cpu_share=0.0, mem_res=9372, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54224, cpu_share=0.01, mem_res=42644, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7594544, free=325228, buffers=427812, cached=7836620, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.57, five_min_avg=3.43, fifteen_min_avg=2.25, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6fc1e8ed6e943c39fd1132e1f8a51ad3e16e79cda3413d2673225d7e48f26868', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 292302} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6075afeda673338dfb2d850ea1914f673da8d45a27ec331c1013a01d914f6741', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 292325} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12527644, partition_space_available_1k=68562840, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1368216, cpu_share=0.04, mem_res=1334200, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13116, cpu_share=0.0, mem_res=9024, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54288, cpu_share=0.01, mem_res=42684, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7565796, free=340948, buffers=428152, cached=7849308, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.8, five_min_avg=3.05, fifteen_min_avg=2.19, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:50:55] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6815 0.001512 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:50:58] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6815 0.001453 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:51:03] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6815 0.001389 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '463b3704691df1166772406f632765f9d8b29db495c71f08f4ad68565fe2d43c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 298133} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'fc6c80614669a653b3e63c7cf9ce7868d40cd38299198cbccb1adf2aed142f9e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 298155} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12539076, partition_space_available_1k=68551408, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1374572, cpu_share=0.05, mem_res=1338792, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13180, cpu_share=0.0, mem_res=9024, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54136, cpu_share=0.01, mem_res=42588, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7578936, free=312128, buffers=428552, cached=7864588, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.61, five_min_avg=3.09, fifteen_min_avg=2.26, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '875b6f540d88042fbedfa583a031ef23f4c23bc8a97fe002fb97817947bbd6bd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 299232} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '47f9127e3fa949e9b3416849dbf8d168a17ec5a255fdad4a2da39a4cf4028240', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 299255} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12546072, partition_space_available_1k=68544412, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1378208, cpu_share=0.04, mem_res=1340744, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13256, cpu_share=0.0, mem_res=9156, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54176, cpu_share=0.01, mem_res=42604, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7587312, free=294720, buffers=428768, cached=7873404, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.88, five_min_avg=2.78, fifteen_min_avg=2.2, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8567d0fe3b81bc9032cae0c5db40810592fca4417b9c472f9d54de6f7a693e79', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 304669} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8ff7aa4f221a078ebddc5a77ad4fb9be623d999e1728b8f33dda9b08fb0a2571', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 304691} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12554172, partition_space_available_1k=68536312, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1380452, cpu_share=0.04, mem_res=1341452, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13340, cpu_share=0.0, mem_res=9156, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54164, cpu_share=0.01, mem_res=42792, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7596432, free=268808, buffers=429120, cached=7889844, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.33, five_min_avg=2.78, fifteen_min_avg=2.24, cpu_share=9.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:53:47] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.001387 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ca0d3d17ae509c6f6cbc5c0c3ee6ccb9a0342c45728b292cf237aa7fc8dc873f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 306382} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '61724f5f2e39344596f7dc1303f361164e728e7feadc0ea036a92d6b4eeb4329', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 306405} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12562444, partition_space_available_1k=68528040, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1384384, cpu_share=0.04, mem_res=1343400, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13416, cpu_share=0.0, mem_res=9288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54224, cpu_share=0.01, mem_res=42828, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7602876, free=251628, buffers=429220, cached=7900480, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.94, five_min_avg=2.78, fifteen_min_avg=2.27, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '356c18937af568061593fd9c6b2195e0c46bf80a5a68a8fe85473192dcee9b9c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 307468} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a53c4167369d62ea2060c9559d25c308acd048c497da94beb0b2bf04c4d218cd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 307507} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12564324, partition_space_available_1k=68526160, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1386852, cpu_share=0.04, mem_res=1344464, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13416, cpu_share=0.0, mem_res=9288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54228, cpu_share=0.01, mem_res=42744, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7606912, free=244544, buffers=429232, cached=7903516, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.01, five_min_avg=2.81, fifteen_min_avg=2.31, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:56:08] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001761 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:56:11] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001183 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:56:15] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001163 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1a53baced245ec1b8da1646411d3d864b9d9f20ebbbd4d03e546a6aa2913e195', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 312981} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '366d104a566d3c852ba82891a17844a8652808785b59c349d482c69dcff6b6ba', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 313004} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12570388, partition_space_available_1k=68520096, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1389968, cpu_share=0.04, mem_res=1345568, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13332, cpu_share=0.0, mem_res=9156, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54208, cpu_share=0.01, mem_res=42660, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7607608, free=330948, buffers=427336, cached=7818312, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.36, five_min_avg=2.66, fifteen_min_avg=2.29, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5c2a754f24abc98ce23a9f30863c6108a971d9de8ca80be766ee1d982dc81d01', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 318767} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '37bfe7c393483ab20fbe7abbfe9f2434cde70052117fc6eb4d83b0294ec6e1ac', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 318788} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12573492, partition_space_available_1k=68516992, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1392228, cpu_share=0.04, mem_res=1346740, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13416, cpu_share=0.0, mem_res=9288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54344, cpu_share=0.01, mem_res=42900, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7618408, free=302132, buffers=428640, cached=7835024, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.54, five_min_avg=2.65, fifteen_min_avg=2.31, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-30 23:58:16] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.001705 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e05af5fe66b6ecd2715daf55dd3d362ab1a9dfac7a9c8f7207cfcb811c8fabdc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 320504} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c6b58a4842736bf52e6f90fd0e264e72716887bb2851f58dd81130f9c467bee3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 320527} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12576920, partition_space_available_1k=68513564, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1395528, cpu_share=0.04, mem_res=1348136, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13460, cpu_share=0.0, mem_res=9288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54316, cpu_share=0.01, mem_res=42784, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7629132, free=330380, buffers=427928, cached=7796764, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.77, five_min_avg=2.71, fifteen_min_avg=2.35, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f093d7bdfcf78604412436fbd84d227bbb67589088ccb1d8d5ee786be2a1de3f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 321619} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0008ac53ae2d2f87dedde907d2820e81089752ece985ba7cfebd5be8c16fa29d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 321640} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12592652, partition_space_available_1k=68497832, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1399304, cpu_share=0.04, mem_res=1350156, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13556, cpu_share=0.0, mem_res=9288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54376, cpu_share=0.01, mem_res=42680, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7636108, free=318452, buffers=428032, cached=7801612, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.13, five_min_avg=2.54, fifteen_min_avg=2.31, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e285eca81f2584a8e70e6d188654453c85b9eda8a655dc4f9a4b51a8397e6248', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 322783} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '39311cc27faea8116b8b3791ab0474546b43219a4d6a8a71f4ab31f9184b8c7b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 322807} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12594700, partition_space_available_1k=68495784, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1402536, cpu_share=0.04, mem_res=1351472, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13608, cpu_share=0.0, mem_res=9420, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54428, cpu_share=0.01, mem_res=42692, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7639636, free=302424, buffers=428748, cached=7813396, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.9, five_min_avg=2.68, fifteen_min_avg=2.37, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:01:20] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001513 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:01:23] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001128 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:01:28] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001189 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8ed04100dbd2ea17cd65652439b972e20ebf5921c8a4ebdb846fe421b6ecdf49', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 328319} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6100043bce0405b4832d697aaff0c994ea2e0a5dcf4e030fb66f0f6db450144a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 328342} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12600448, partition_space_available_1k=68490036, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1404800, cpu_share=0.04, mem_res=1352160, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13664, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54424, cpu_share=0.01, mem_res=42692, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7643436, free=290668, buffers=428852, cached=7821248, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.49, five_min_avg=2.6, fifteen_min_avg=2.36, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '98066db690efc0709795853a0fc7e1270027f36f7f55644881066577ecb67537', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 330715} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0632ebb4dc7238d2b11bfe1ece6d76cdd3db8374a81091ee0b35caa47db76218', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 330749} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12603020, partition_space_available_1k=68487464, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1408148, cpu_share=0.04, mem_res=1353904, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13568, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54564, cpu_share=0.01, mem_res=42892, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7708288, free=218532, buffers=429220, cached=7828164, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.13, five_min_avg=2.44, fifteen_min_avg=2.32, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd7fb9c66b81155aac9c3b08745da0c6565866fbac182cf3a1bb5c73f7d8d9173', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 335183} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'dd9e1a132f2d10cb2d919f0e7673810d22d4298412b2c2e671b720ff4a8640ce', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 335208} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12605028, partition_space_available_1k=68485456, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1411116, cpu_share=0.03, mem_res=1355200, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13576, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54600, cpu_share=0.01, mem_res=42748, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7651992, free=266596, buffers=429608, cached=7836008, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.08, five_min_avg=2.4, fifteen_min_avg=2.31, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:04:07] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001716 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'da5a0b7c5fc597fea57b207f53fa717eb2fdec230920362b315a4a38b3a4bf90', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 336925} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '213b063f12b869afee20440ea401668313bc768069ff2df014633b709c298025', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 336950} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12608136, partition_space_available_1k=68482348, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1415664, cpu_share=0.04, mem_res=1357652, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13620, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54308, cpu_share=0.01, mem_res=42836, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7657340, free=256820, buffers=429652, cached=7840392, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.86, five_min_avg=2.25, fifteen_min_avg=2.27, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'abe96e941c174385bdd220e41b020b63e508dabfb92e504df9d5de6a7cd9a0fa', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 338009} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '454b1a3c1604190a5b50784bdbb3194c750dcd0c6143c4deb78641de025eff2e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 338035} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12615204, partition_space_available_1k=68475280, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1419408, cpu_share=0.04, mem_res=1359784, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13620, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54332, cpu_share=0.01, mem_res=42812, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7653160, free=252496, buffers=429692, cached=7848856, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.0, five_min_avg=2.21, fifteen_min_avg=2.25, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '524d5d4a5686eb995e2bd989cd4559e802b22f7f15e0072d9a9d3e60b7edd142', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 339107} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cef07306c7f7ed50d665b91b7053aa745dcd03b8b781b1d213cf8a907eb23d43', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 339132} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12616988, partition_space_available_1k=68473496, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1422928, cpu_share=0.03, mem_res=1361424, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13620, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54340, cpu_share=0.01, mem_res=42848, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7657896, free=244732, buffers=429700, cached=7851876, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.75, five_min_avg=2.11, fifteen_min_avg=2.21, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:06:44] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.005518 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:06:47] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.001455 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:06:51] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.001688 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a849119d5d1cb8243a5202fe6cdecb2e0f1b95ecd7e93e2abca559ca2e48e548', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 344915} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '87515a25ced393ecbcf7049cf7b4f807e8d0504bef1fb5e1d2ccb2de7b4eb759', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 344940} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12627664, partition_space_available_1k=68462820, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1426184, cpu_share=0.04, mem_res=1363348, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13740, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54416, cpu_share=0.01, mem_res=42684, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7667636, free=287700, buffers=428824, cached=7800044, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.64, five_min_avg=2.0, fifteen_min_avg=2.17, cpu_share=9.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '087b5b4971afe85ffe66ce1b68501414c3ee4f1ec4992d17333aba3d67ea79de', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 347834} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e5f7edb3cfb108d574eb07fb3c2f848cd0df29ea24dd303cb08c40a083112de7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 347901} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12620672, partition_space_available_1k=68469812, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1429172, cpu_share=0.04, mem_res=1365092, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13636, cpu_share=0.0, mem_res=9552, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54512, cpu_share=0.01, mem_res=42624, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7735956, free=244692, buffers=427980, cached=7775576, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.24, five_min_avg=2.07, fifteen_min_avg=2.18, cpu_share=9.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:09:02] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001694 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c5cc361401bb91c0abeed01a1a4e677f0319e00b6146c397552cc199298f3f6e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 352117} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c56d36c3d8ee6e41a18bda23d89d5c5a6baaeee0272da50b20b8a2258e03aeb2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 352140} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12624468, partition_space_available_1k=68466016, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1433740, cpu_share=0.04, mem_res=1367616, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13696, cpu_share=0.0, mem_res=9684, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54528, cpu_share=0.01, mem_res=42660, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7693052, free=277428, buffers=428444, cached=7785280, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.89, five_min_avg=2.01, fifteen_min_avg=2.15, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a6e67a156cd7f34d15308e9d4425b9ada4da9c8bde3f8a229d7acd0ae96c1082', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 353206} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '41a34fe93b0e30577ad5543e4193cd9a51ec84afe7b1b4f2af9f3d95892c30e5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 353231} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12626608, partition_space_available_1k=68463876, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1437152, cpu_share=0.04, mem_res=1369044, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13700, cpu_share=0.0, mem_res=9684, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54636, cpu_share=0.01, mem_res=42708, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7694140, free=272980, buffers=428448, cached=7788636, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.44, five_min_avg=1.85, fifteen_min_avg=2.09, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd8324e1e365bb27ff1705b79eec2fc77a74166bad7c16232b6034306e83504eb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 354296} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3fd5f03683be05ea64066c2a3da606d0dc15f892143632051f49757718ba699b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 354319} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12633316, partition_space_available_1k=68457168, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1440596, cpu_share=0.03, mem_res=1371060, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13704, cpu_share=0.0, mem_res=9684, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54528, cpu_share=0.01, mem_res=42636, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7697324, free=262040, buffers=428484, cached=7796356, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.79, five_min_avg=1.84, fifteen_min_avg=2.07, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:11:38] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.001433 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:11:41] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.001476 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:11:45] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6817 0.001263 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2f89ddcf65114496b64561ed485e8a08143aa66a3a76dda2a2ab5ca55ef9273b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 359800} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8dde0278d4ca131cb20c0b93201815832e30340479f8bd6dbd84f5d0dd5167cd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 359822} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12639240, partition_space_available_1k=68451244, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1443688, cpu_share=0.04, mem_res=1372484, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13784, cpu_share=0.0, mem_res=9684, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54548, cpu_share=0.01, mem_res=42588, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7700488, free=250280, buffers=428692, cached=7804744, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.59, five_min_avg=1.8, fifteen_min_avg=2.04, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '22dde5652647d2daf2d1023ad62521f2729486f2b98d11eaee533680c182c5a7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 361238} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'efdb538a2ab303190ac9e2f317eeaccfde34068f11f6f25b8c8559a08eac2f93', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 361262} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12641676, partition_space_available_1k=68448808, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1447000, cpu_share=0.04, mem_res=1374256, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13828, cpu_share=0.0, mem_res=9816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54616, cpu_share=0.01, mem_res=42796, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7741464, free=202584, buffers=429016, cached=7811140, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.52, five_min_avg=1.74, fifteen_min_avg=2.0, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7b6123ca91caa91f92979652505bedf76c8847ab7eb56abdc044c897da8e8f4d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 367243} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:14:31] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6815 0.001397 CRITICAL:root:Exit data: {'ID': '4fd9c69bf63efaf4fa3ef883265e542c01edbb43a2a74804bfa7ef66a9b8c060', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 367265} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12650956, partition_space_available_1k=68439528, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1450628, cpu_share=0.04, mem_res=1376632, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13724, cpu_share=0.0, mem_res=9816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54580, cpu_share=0.01, mem_res=42908, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7716532, free=316220, buffers=426344, cached=7725108, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.53, five_min_avg=1.93, fifteen_min_avg=2.05, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4c6aa6b6abcf34182e310d43e97339d83563bd6c4c906edc1bf6ad0ae1ff8f07', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 368391} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7ceb93afef61cf296df0547ece8a2155d903fe0d816ecf3cc285d62cc16979c1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 368413} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12648048, partition_space_available_1k=68442436, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1453844, cpu_share=0.04, mem_res=1378272, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13772, cpu_share=0.0, mem_res=9816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54532, cpu_share=0.01, mem_res=42712, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7720608, free=305068, buffers=426608, cached=7731920, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.93, five_min_avg=1.85, fifteen_min_avg=2.01, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '41852c335d2ddd55259c2a70e1ada6cc14bcfbb3c1dff9f5161e18ffde705294', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 369478} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e43781166fdc8569a6e00dc03d1dc8a94ebc3ffd44c3a76fdc93af397d0aa44e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 369501} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12649856, partition_space_available_1k=68440628, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1456972, cpu_share=0.04, mem_res=1380000, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13776, cpu_share=0.0, mem_res=9816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54468, cpu_share=0.01, mem_res=42516, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7717692, free=304940, buffers=426652, cached=7734920, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.89, five_min_avg=1.84, fifteen_min_avg=1.99, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:17:18] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001486 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:17:21] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001180 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:17:25] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001203 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b505ba9a55335e00fa301fff6febb0fb34abef0f8810c2245a3877421beec9a1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 374981} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b1d82794f0e7b37590b4c173c02288fe611ff9b43cf49dd83aa05d969d442832', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 375004} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12660452, partition_space_available_1k=68430032, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1460404, cpu_share=0.04, mem_res=1381724, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13800, cpu_share=0.0, mem_res=9816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54452, cpu_share=0.01, mem_res=42576, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7728300, free=280624, buffers=426708, cached=7748572, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.32, five_min_avg=1.97, fifteen_min_avg=2.03, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9a9d938af4b91c003bfef0da7d1232296c6fb1e93f4e5f166ef8acfd8248f1af', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 376459} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '04659d948cfc35006f3590bea7e5978d9252d1b230c3dc6864686b2472e8d373', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 376481} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12662472, partition_space_available_1k=68428012, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1463772, cpu_share=0.04, mem_res=1383368, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13812, cpu_share=0.0, mem_res=9816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54700, cpu_share=0.01, mem_res=42488, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7674340, free=330816, buffers=426712, cached=7752336, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.12, five_min_avg=2.01, fifteen_min_avg=2.04, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:19:28] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001458 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0c7ac575849bed77093c76ed04b49aab0f7119ece654ee6d10108077296b9482', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 382528} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'bb242ce246d5e4572d25bd9b548a2a7264012af7024718a3568c841c8f7feb19', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 382550} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12666696, partition_space_available_1k=68423788, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1468212, cpu_share=0.04, mem_res=1385788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13820, cpu_share=0.0, mem_res=9816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54804, cpu_share=0.01, mem_res=42620, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7678924, free=312248, buffers=427404, cached=7765628, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.19, five_min_avg=2.1, fifteen_min_avg=2.07, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd1bab80b66fb868509d75b32a96f5261bf86249ab39a7a994b03fe287399c26c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 383611} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '79b5300f485fa99c5c5dcb96881135aa372b50c790a329890f7deab3b32f2e08', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 383633} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12676784, partition_space_available_1k=68413700, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1471332, cpu_share=0.04, mem_res=1387192, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13724, cpu_share=0.0, mem_res=9672, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54724, cpu_share=0.01, mem_res=42844, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7680272, free=307724, buffers=427428, cached=7768780, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.44, five_min_avg=1.9, fifteen_min_avg=2.0, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0379ba9ee6b75e510043a13adc47e9c27080d8f3cb9564ec6cc5f30dfe35baf9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 384704} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5d07569157c5934c32d0d093ffbbb8cd5408bd00190baa49a06dd38e61a70949', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 384726} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12683008, partition_space_available_1k=68407476, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1475028, cpu_share=0.04, mem_res=1389372, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13728, cpu_share=0.0, mem_res=9672, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54560, cpu_share=0.01, mem_res=42784, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7680024, free=300724, buffers=427444, cached=7776012, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.99, five_min_avg=1.98, fifteen_min_avg=2.02, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1dc59a27ac0acfb9eb92ba5b374dae8836e3233aeef6feba71463b56bcce6e86', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 386133} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a8287e513aae040d4ceab9f8bfab68f5c1f07434f901066e783df0e99040ca73', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 386156} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12675152, partition_space_available_1k=68415332, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1478292, cpu_share=0.04, mem_res=1391028, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13800, cpu_share=0.0, mem_res=9804, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54460, cpu_share=0.01, mem_res=42568, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7679092, free=308108, buffers=427592, cached=7769412, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.45, five_min_avg=2.1, fifteen_min_avg=2.06, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:23:09] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001385 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:23:12] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001259 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:23:16] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001454 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '93521073f40f2db11f1599fd330c8822a7d5be8722adb65e2ac45054df3734bc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 396535} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.49 - - [2025-08-31 00:23:30] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6816 0.001171 CRITICAL:root:Exit data: {'ID': '762e09269308658a5343464f09ada0520af430b85abd9f6398ef017e35513aa9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 396560} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12692324, partition_space_available_1k=68398160, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1482896, cpu_share=0.04, mem_res=1393792, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13844, cpu_share=0.0, mem_res=9936, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54540, cpu_share=0.01, mem_res=42852, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7694580, free=309264, buffers=426172, cached=7754188, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.96, five_min_avg=2.5, fifteen_min_avg=2.2, cpu_share=11.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '646b6b5537ac07b6b700fc39fbba650c02f639366797a18ce1c3a8fd4f0f32b0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 397715} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '505bb3660585e244b46e4f237e2e2cb71549042db415941d7c8467a1b5e10576', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 397739} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12690744, partition_space_available_1k=68399740, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1486212, cpu_share=0.04, mem_res=1395588, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13844, cpu_share=0.0, mem_res=9936, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54648, cpu_share=0.01, mem_res=42828, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7694156, free=304408, buffers=426384, cached=7759256, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.27, five_min_avg=2.29, fifteen_min_avg=2.14, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '78a97799f52a26a052722800fa6411f3696d1aadd3b246997e39f22a359ea514', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 398800} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1085cd182869e7c1149b2b670b6e73093bd7693e6c87137cce9a6f718d39697f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 398822} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12692688, partition_space_available_1k=68397796, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1489168, cpu_share=0.04, mem_res=1396904, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13856, cpu_share=0.0, mem_res=9936, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54740, cpu_share=0.01, mem_res=42844, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7694996, free=300884, buffers=426400, cached=7761924, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.38, five_min_avg=2.02, fifteen_min_avg=2.06, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ab08a586bcf196d070920ffb39d4fd9fc31f767b705dffc3c45b601298621f1e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.49', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 400245} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f357f4f153f0221caed11ecb021358bdcec3356539f7030a1fd86dbf0d50b989', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '53a82653f8cadd6691630263d8605300af949bb291c6e8161226d9a28b9738c5', 'DetachKeys': '', 'Pid': 400267} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1450-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12694564, partition_space_available_1k=68395920, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1492544, cpu_share=0.04, mem_res=1398644, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13808, cpu_share=0.0, mem_res=9936, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54972, cpu_share=0.01, mem_res=42944, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184204, used=7701052, free=291596, buffers=426548, cached=7765008, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.36, five_min_avg=1.91, fifteen_min_avg=2.01, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo')