++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ mkdir -p /var/log/contrail/database-nodemgr ++ log_file=/var/log/contrail/database-nodemgr/console.log ++ touch /var/log/contrail/database-nodemgr/console.log ++ chmod 600 /var/log/contrail/database-nodemgr/console.log ++ exec +++ tee -a /var/log/contrail/database-nodemgr/console.log +++ date ++ echo 'INFO: =================== Thu Aug 28 01:44:00 AM UTC 2025 ===================' INFO: =================== Thu Aug 28 01:44:00 AM UTC 2025 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ ip route get 1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ cut -d / -f 1 +++ ip addr show dev ens3 +++ awk '{print $2}' +++ grep 'inet ' +++ head -n 1 ++ DEFAULT_LOCAL_IP=10.0.0.50 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=0 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 0 ]] ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=kubernetes ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=no-auth ++ AUTH_MODE=noauth ++ AUTH_PARAMS= ++ SSL_ENABLE=False ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.50 ++ ANALYTICS_ALARM_ENABLE=True ++ ANALYTICS_SNMP_ENABLE=True ++ ANALYTICSDB_ENABLE=True ++ ANALYTICS_NODES=10.0.0.50 ++ ANALYTICSDB_NODES=10.0.0.50 ++ ANALYTICS_SNMP_NODES=10.0.0.50 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:8081 ' +++ '[' -n '10.0.0.50:8081 ' ']' +++ echo 10.0.0.50:8081 ++ ANALYTICS_SERVERS=10.0.0.50:8081 +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9042 ' +++ '[' -n '10.0.0.50:9042 ' ']' +++ echo 10.0.0.50:9042 ++ ANALYTICSDB_CQL_SERVERS=10.0.0.50:9042 ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.50 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:8086 ' +++ '[' -n '10.0.0.50:8086 ' ']' +++ echo 10.0.0.50:8086 ++ COLLECTOR_SERVERS=10.0.0.50:8086 ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.50 ++ CONFIGDB_NODES=10.0.0.50 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:8082 ' +++ '[' -n '10.0.0.50:8082 ' ']' +++ echo 10.0.0.50:8082 ++ CONFIG_SERVERS=10.0.0.50:8082 +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9161 ' +++ '[' -n '10.0.0.50:9161 ' ']' +++ echo 10.0.0.50:9161 ++ CONFIGDB_SERVERS=10.0.0.50:9161 +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9041 ' +++ '[' -n '10.0.0.50:9041 ' ']' +++ echo 10.0.0.50:9041 ++ CONFIGDB_CQL_SERVERS=10.0.0.50:9041 ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=False ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=False ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=False ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.0.0.50 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.0.0.50 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.50 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+=10.0.0.50:2181, +++ '[' -n 10.0.0.50:2181, ']' +++ echo 10.0.0.50:2181 ++ ZOOKEEPER_SERVERS=10.0.0.50:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:2181 ' +++ '[' -n '10.0.0.50:2181 ' ']' +++ echo 10.0.0.50:2181 ++ ZOOKEEPER_SERVERS_SPACE_DELIM=10.0.0.50:2181 ++ RABBITMQ_NODES=10.0.0.50 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+=10.0.0.50:5673, +++ '[' -n 10.0.0.50:5673, ']' +++ echo 10.0.0.50:5673 ++ RABBITMQ_SERVERS=10.0.0.50:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=False ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:6379 ' +++ '[' -n '10.0.0.50:6379 ' ']' +++ echo 10.0.0.50:6379 ++ REDIS_SERVERS=10.0.0.50:6379 ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=False ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.50 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.50 +++ local server_address=10.0.0.50 +++ extended_server_list+='10.0.0.50:9092 ' +++ '[' -n '10.0.0.50:9092 ' ']' +++ echo 10.0.0.50:9092 ++ KAFKA_SERVERS=10.0.0.50:9092 ++ KAFKA_SSL_ENABLE=False ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/ ++ KEYSTONE_AUTH_HOST=127.0.0.1 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.50 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.50 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=False ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=False ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=False ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ / == \/\v\2\.\0 ]] ++ [[ kubernetes == \o\p\e\n\s\t\a\c\k ]] ++ [[ noauth == \k\e\y\s\t\o\n\e ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ noauth == \k\e\y\s\t\o\n\e ]] + cat + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + [[ database == \t\o\r\a\g\e\n\t ]] + export NODEMGR_TYPE=contrail-database + NODEMGR_TYPE=contrail-database + NODEMGR_NAME=contrail-database-nodemgr ++ echo DATABASE ++ tr - _ + ntype=DATABASE + [[ DATABASE == \V\R\O\U\T\E\R ]] + [[ DATABASE == \D\A\T\A\B\A\S\E ]] + htype=ANALYTICSDB + [[ DATABASE == \C\O\N\T\R\O\L ]] ++ get_listen_ip_for_node ANALYTICSDB +++ find_my_ip_and_order_for_node ANALYTICSDB +++ local server_typ=ANALYTICSDB_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.50 +++ local servers=10.0.0.50 +++ local server_list= +++ IFS=, +++ read -ra server_list +++ cut -d ' ' -f 1 ++++ get_local_ips ++++ tr '\n' , ++++ uniq ++++ sort ++++ grep -vi host ++++ awk '/32 host/ { print f } {f=$2}' ++++ cat /proc/net/fib_trie +++ local local_ips=,10.0.0.50,10.68.75.1,127.0.0.1,172.17.0.1,252.50.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.50'\''))' +++ local server_ip=10.0.0.50 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.50 ]] +++ [[ ,10.0.0.50,10.68.75.1,127.0.0.1,172.17.0.1,252.50.0.1,, =~ ,10\.0\.0\.50, ]] +++ echo 10.0.0.50 1 +++ return ++ local ip=10.0.0.50 ++ [[ -z 10.0.0.50 ]] ++ echo 10.0.0.50 + hostip=10.0.0.50 + introspect_ip=0.0.0.0 + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + mkdir -p /etc/contrail + cat ++ format_boolean False ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ echo False + '[' -n '' ']' + cat + add_ini_params_from_env DATABASE_NODEMGR /etc/contrail/contrail-database-nodemgr.conf + local service_name=DATABASE_NODEMGR + local cfg_path=/etc/contrail/contrail-database-nodemgr.conf + local delim=__ ++ grep '^DATABASE_NODEMGR__.*__.*=.*$' ++ set -o posix ++ set ++ sort ++ cut -d = -f 1 ++ sed 's/^DATABASE_NODEMGR__//g' + local vars=DEFAULTS__minimum_diskGB + local section= + for var in $vars + local var_name=DATABASE_NODEMGR__DEFAULTS__minimum_diskGB + local val=4 ++ sed s/DASH/-/g ++ echo DEFAULTS__minimum_diskGB + local var_unsubst=DEFAULTS__minimum_diskGB ++ echo DEFAULTS__minimum_diskGB ++ sed 's/^\(.*\)__.*$/\1/' + local var_section=DEFAULTS + [[ '' != \D\E\F\A\U\L\T\S ]] + echo '[DEFAULTS]' + section=DEFAULTS ++ echo DEFAULTS__minimum_diskGB ++ sed 's/.*__\(.*\)$/\1/' + local var_param=minimum_diskGB + echo 'minimum_diskGB = 4' + cat /etc/contrail/contrail-database-nodemgr.conf [DEFAULTS] http_server_ip=0.0.0.0 log_file=/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log log_level=SYS_DEBUG log_local=1 hostip=10.0.0.50 db_port=9042 db_jmx_port=7200 db_use_ssl=False [COLLECTOR] server_list=10.0.0.50:8086 [SANDESH] introspect_ssl_enable=False sandesh_ssl_enable=False [DEFAULTS] minimum_diskGB = 4 + upgrade_old_logs contrail-database-nodemgr + local template=contrail-database-nodemgr ++ dirname /var/log/contrail/database-nodemgr + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/database-nodemgr + chmod 755 /var/log/contrail/database-nodemgr + mv -n '/var/log/contrail/*contrail-database-nodemgr.log*' /var/log/contrail/database-nodemgr/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/database-nodemgr + find /var/log/contrail/database-nodemgr -uid 0 -exec chown 1999:1999 '{}' + + exec /bin/sh -c '/usr/local/bin/contrail-nodemgr --nodetype=${NODEMGR_TYPE}' 08/28/2025 01:44:05.688 7f538d638880 [contrail-database-nodemgr] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 08/28/2025 01:44:06.208 7f538d638880 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 01:44:06.255 7f538d638880 [contrail-database-nodemgr] [INFO]: SANDESH: INTROSPECT IS ON: 0.0.0.0:8103 08/28/2025 01:44:06.279 7f538d638880 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 01:44:06.303 7f538d638880 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 08/28/2025 01:44:06.303 7f538d638880 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log] Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: CRITICAL:root:Exit data: {'ID': 'd10e0ec55b9324c99731d42269db83af29ebf78f88be592de3274f3d0e885aae', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 274333} ERROR:contrail-database-nodemgr:Failed to connect to database by CQL and socket: Result: 1 INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1433-1', _node_type='Database', _instance_id='0', _timestamp=1756345448918927, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Non-Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=[''], status='Down', description='none to Idle on EvStart')]), flag_infos=None, description='Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1433-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-database-nodemgr:Session Event: TCP Connected INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-k8s-1433-1 process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.50:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001014 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[ClientInit] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 0 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Client: Event[EvSandeshCtrlMessageRecv] => State[ClientInit] -> State[Established] INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] CRITICAL:root:Exit data: {'ID': '6396c090aa690b655e1b70e161b208c9fd4e106bbd843d10fa286451281efb23', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 274354} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 CRITICAL:root:Exit data: {'ID': 'ca4dfee4c812a055ad9bf11557cd7bc297b019784e6ce62a2a587c17d712f0c6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'repair', '-pr', 'ContrailAnalyticsCql'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 275132} ERROR:contrail-database-nodemgr:Failed to run cmd: 'nodetool -p 7200 repair -pr ContrailAnalyticsCql'. Error is: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:44:19] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 4466 0.001715 INFO:contrail-database-nodemgr:send_process_state_db: Sending UVE: NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1433-1', _node_type='Database', _instance_id='0', _timestamp=1756345461985320, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=False, status=None, process_status=None, process_info=[ProcessInfo(process_name='cassandra', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756345440000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-query-engine', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756345440000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-database-nodemgr', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756345440000000', last_stop_time='', last_exit_time='', core_file_list=[])], disk_usage_info=None, description=None, all_core_file_list=None, build_info='{"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-08-28 00:14:38.756636", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10583-1.novalocal", "build-id" : "v0+nightly", "build-number" : "nightly.el9"}]}', process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12431216, partition_space_available_1k=68659268, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=[], build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1260756, cpu_share=0.0, mem_res=1232008, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=9696, cpu_share=0.0, mem_res=5532, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55328, cpu_share=0.0, mem_res=42884, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7184516, free=189196, buffers=404612, cached=8405884, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=8.75, five_min_avg=3.69, fifteen_min_avg=1.69, cpu_share=0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:44:32] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6783 0.001395 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:44:45] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6783 0.002735 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:44:58] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6783 0.001332 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:45:04] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6783 0.001328 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c41bdfa138001f2a358e8ad7d4561c9239ab42858ad6d296fe3390406ae77359', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 284482} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-k8s-1433-1', _node_type='Database', _instance_id='0', _timestamp=1756345508131567, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=['10.0.0.50:8086'], status='Up', description='ClientInit to Established on EvSandeshCtrlMessageRecv')]), flag_infos=None, description='')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:45:08] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6748 0.001511 CRITICAL:root:Exit data: {'ID': '016ee72b4076a0282e2d5b60b592b1b3ed5b6a0f270c101ae8252d3d96a40bf6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 284505} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12477716, partition_space_available_1k=68612768, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1361548, cpu_share=0.42, mem_res=1322624, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13324, cpu_share=0.0, mem_res=8600, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55348, cpu_share=0.01, mem_res=43064, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7455204, free=338540, buffers=411036, cached=7979428, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=15.48, five_min_avg=6.36, fifteen_min_avg=2.71, cpu_share=20.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:45:30] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6755 0.001337 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '294d560f20b9d941071e3ada2556e3deaa7f0ba190dc5f171614bcc4dec68047', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 286293} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '58a224a4802e77e589a3b4c2d02e3425573dcf3ccb53db11d1475a28867599c8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 286315} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12480980, partition_space_available_1k=68609504, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1365088, cpu_share=0.07, mem_res=1324808, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13644, cpu_share=0.0, mem_res=8864, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55424, cpu_share=0.01, mem_res=42968, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7469704, free=302316, buffers=411948, cached=8000240, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=6.96, five_min_avg=5.59, fifteen_min_avg=2.68, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '88f9f281dfa148a83bea9622b3c1664ed6d377281f0f6728f0fe9b70eb67cdbd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 287393} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cf390db13148dca9a6f6e4e77823c3dd9fe25466120b787e6dbacf6bfd0cb5e8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 287415} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12488012, partition_space_available_1k=68602472, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1375620, cpu_share=0.07, mem_res=1333392, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13744, cpu_share=0.0, mem_res=8864, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55560, cpu_share=0.01, mem_res=43016, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7471016, free=291796, buffers=412068, cached=8009328, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.34, five_min_avg=4.81, fifteen_min_avg=2.59, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:48:04] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001779 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:48:07] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001278 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dea544426d767ad9ce2785d1f7a692512c0a00954ba80f8af256d0c441ad6ab1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 295023} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a4a98f784ddbcb49ec8df8df910c22619d434b05d4074a80aa527627727b6bad', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 295044} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:48:11] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001888 DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12498324, partition_space_available_1k=68592160, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1377848, cpu_share=0.06, mem_res=1334260, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13860, cpu_share=0.0, mem_res=9128, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55628, cpu_share=0.01, mem_res=42972, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7548188, free=253268, buffers=425608, cached=7957144, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.99, five_min_avg=4.89, fifteen_min_avg=2.75, cpu_share=12.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e21aefb82a7300b1fea98c2667de52c0121ff0c2ad397df6f6b921ac9d25b18a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 298366} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7a64f002f1a76eb34c3fd4d33b59a83498e77c604a0f9d0a276fdd10b1747e6d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 298395} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12507484, partition_space_available_1k=68583000, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1381144, cpu_share=0.06, mem_res=1336104, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13920, cpu_share=0.0, mem_res=9128, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55812, cpu_share=0.01, mem_res=42836, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7553728, free=232132, buffers=425824, cached=7972524, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.79, five_min_avg=4.26, fifteen_min_avg=2.66, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c1e8049224775c0a995be74357dc19427a02b3ce9b9aff3533e4eb5ab3c97094', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 299476} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '36ba31f27892c81ef235e3810ca5aa967458f46e5aa997b4fb46b4194bcc9ab6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 299497} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12515284, partition_space_available_1k=68575200, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1384680, cpu_share=0.06, mem_res=1337896, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14020, cpu_share=0.0, mem_res=9260, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55960, cpu_share=0.01, mem_res=42776, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7516756, free=284616, buffers=424648, cached=7958188, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.97, five_min_avg=3.74, fifteen_min_avg=2.58, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:51:03] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001382 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ed20b4e2f41950c6f902df60f46342882731ab02ddb2cc86678c1408f1997010', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 301545} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c812ae1de0a21ddce79de86f982031f78d9386003d07e740fe804466af0ab5bd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 301568} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12523016, partition_space_available_1k=68567468, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1386972, cpu_share=0.05, mem_res=1337996, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13656, cpu_share=0.0, mem_res=9044, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55944, cpu_share=0.01, mem_res=42896, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7523856, free=263576, buffers=424860, cached=7971916, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.88, five_min_avg=3.39, fifteen_min_avg=2.54, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '24c32eda9565d7fbdd5591bec4c9b6ca449429bdde95eb0242857087128fe314', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 308492} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:52:07] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001521 CRITICAL:root:Exit data: {'ID': '2b201b7afbe6302bc0e7210df4d44892184e895c5f4496798d565d3f78fecc21', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 308525} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:52:10] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001650 DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12533372, partition_space_available_1k=68557112, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1388976, cpu_share=0.04, mem_res=1338784, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13740, cpu_share=0.0, mem_res=9044, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55848, cpu_share=0.01, mem_res=42924, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7602636, free=209964, buffers=424824, cached=7946784, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.41, five_min_avg=3.43, fifteen_min_avg=2.59, cpu_share=11.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:52:14] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6754 0.001388 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '84912fdd71a43e92f292e0004770a24a3515c48bf88c305b3707c310c0f41d4b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 312508} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3710082760a1f177a387ca90807eaeb82a36d6f301af3b5a562395ce9a1ad082', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 312530} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12542920, partition_space_available_1k=68547564, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1393188, cpu_share=0.04, mem_res=1340604, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13864, cpu_share=0.0, mem_res=9176, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55908, cpu_share=0.01, mem_res=42744, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7548736, free=252824, buffers=425340, cached=7957308, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.04, five_min_avg=3.25, fifteen_min_avg=2.58, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0df6a04f55dc4e22da6081b4260b4f1a4cccea7e5965f39818f69275ecac9af3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 313614} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f164d17a049cca634f66bee7cb5d27513e9da4f89e8a5f14f8fa5f34b8003195', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 313635} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12549708, partition_space_available_1k=68540776, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1395580, cpu_share=0.04, mem_res=1341832, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13880, cpu_share=0.0, mem_res=9176, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56048, cpu_share=0.01, mem_res=43088, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7555012, free=237740, buffers=425352, cached=7966104, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.97, five_min_avg=3.12, fifteen_min_avg=2.57, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd44f1b1b00e653c3f3327de8bd79e6704bb7c687f5b64ed5dc8a498af8eae287', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 314705} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '98cd71dbcb7d71082640877359ca0b480b3ac2c377a9f9fd22cd0c571e335993', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 314728} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12557492, partition_space_available_1k=68532992, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1399052, cpu_share=0.05, mem_res=1343228, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13892, cpu_share=0.0, mem_res=9176, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56000, cpu_share=0.01, mem_res=43108, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7556116, free=227428, buffers=425360, cached=7975304, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.57, five_min_avg=3.24, fifteen_min_avg=2.65, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '20ec2c4913dc8ac3d1d8988b34a66521f80c90fc588226d0d5db7f63efa025ef', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 316124} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9f3e4ea878d9bb60588eaae66f8168c03c607b9c0a94f99e6a04df315132e503', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 316147} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12564268, partition_space_available_1k=68526216, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1401672, cpu_share=0.04, mem_res=1344044, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13968, cpu_share=0.0, mem_res=9176, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56092, cpu_share=0.01, mem_res=43208, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7554916, free=220140, buffers=425388, cached=7983764, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.0, five_min_avg=3.15, fifteen_min_avg=2.65, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:56:58] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.001313 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:57:01] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.001270 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:57:06] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.001331 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cdc345d22c588bc84bacab117fed39e3eabed493052b3303c5141fc2ab98ec0b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 325945} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8f212337d4f3aafa76b79cd2b28c594b053b2ee89216a40a758591f9f7cb2caf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 325996} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 01:57:09] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.002033 DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12567456, partition_space_available_1k=68523028, percentage_partition_space_used=15, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1403880, cpu_share=0.04, mem_res=1345432, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13964, cpu_share=0.0, mem_res=9176, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56008, cpu_share=0.01, mem_res=43096, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7589360, free=288548, buffers=423156, cached=7883144, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.78, five_min_avg=3.29, fifteen_min_avg=2.72, cpu_share=11.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a33807103ab5b3790d0b6c18b601d57d9c7c90beb8a9cea509ffaa25739d5d7c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 327663} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '978827b28296c913d4ba5681430c7a6628499b6577531f0dfea30005da917cf4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 327687} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12581516, partition_space_available_1k=68508968, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1406480, cpu_share=0.04, mem_res=1346596, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13972, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55936, cpu_share=0.01, mem_res=43024, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7566976, free=302024, buffers=423844, cached=7891364, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.49, five_min_avg=3.0, fifteen_min_avg=2.66, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd5fdc3475b78d4132e7a37a85e255d17ec1071339b4b55e2c48cb724665784f3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 328764} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6dfb4c3a9e927ddb5a792418e2cacbc4f88aee0e7768295be45aec61e7e8c569', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 328785} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12583492, partition_space_available_1k=68506992, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1409372, cpu_share=0.04, mem_res=1347668, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13984, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55968, cpu_share=0.01, mem_res=42976, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7575656, free=284852, buffers=423868, cached=7899832, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.06, five_min_avg=2.79, fifteen_min_avg=2.61, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a5d94654d2f4488e1e99f8650fcce2dcc8e2cbf5a40dc4f11c18fb8f7e99f41c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 329866} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'fb965a8ff6d08660c79ea5b4817e03e96fa33728d4821e068b6e1bd18511f962', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 329891} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12586456, partition_space_available_1k=68504028, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1414948, cpu_share=0.04, mem_res=1351076, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13984, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55828, cpu_share=0.01, mem_res=42904, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7582044, free=274252, buffers=423932, cached=7903980, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.06, five_min_avg=2.62, fifteen_min_avg=2.56, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:01:02] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6754 0.001903 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:01:05] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6754 0.001324 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b19591b09f8745b72801cb7a595bab31b57651f6f8a3ec79f8b7aa378e8554f7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 338590} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1f68936e34fad6a4baf6a73cb34f7759d6acc0a4658fca41fec0629d8c9ad685', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 338923} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:01:10] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6754 0.001952 DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12594408, partition_space_available_1k=68496076, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1417920, cpu_share=0.04, mem_res=1352496, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14008, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55956, cpu_share=0.01, mem_res=43056, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7589792, free=300388, buffers=423336, cached=7870692, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.76, five_min_avg=2.94, fifteen_min_avg=2.67, cpu_share=11.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:01:57] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6755 0.001109 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b828497ad58484424b21af5bc91c2544abf0a1c410c83f01b77abc69de173f8f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 341790} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '66b3ebb5bf1dc9ac94e9dd93b49912d5d6ea069b4cb08da54b5a53716bc68e7b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 341811} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12595676, partition_space_available_1k=68494808, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1421180, cpu_share=0.04, mem_res=1354288, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14016, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55752, cpu_share=0.01, mem_res=42992, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7590000, free=291732, buffers=423828, cached=7878648, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.56, five_min_avg=2.95, fifteen_min_avg=2.68, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b5486a76cf769a111aecaa9cc7ff92b3cec5902c5ada699295150073121040b4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 342889} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cb25c41505e1a3975fcad6c7c1a546d8f488f876b6258a57b895459b179135fe', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 342910} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12597576, partition_space_available_1k=68492908, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1424068, cpu_share=0.04, mem_res=1355900, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13956, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55808, cpu_share=0.01, mem_res=42944, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7593032, free=285744, buffers=423872, cached=7881560, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.4, five_min_avg=2.72, fifteen_min_avg=2.62, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4574bde15f4823f41a5e9bae3511664808e561cf87f7ffb0221652d516e21fe7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 343996} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cb627c53bee6f02f21d5d8fbb02f96577af37a72f47a23d44a9f0e94b2ee69d2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 344018} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12604496, partition_space_available_1k=68485988, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1430016, cpu_share=0.04, mem_res=1360020, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13960, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55892, cpu_share=0.01, mem_res=42956, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7596344, free=279400, buffers=423884, cached=7884580, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.08, five_min_avg=2.57, fifteen_min_avg=2.57, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6b0cf653895015c12c5b1b851ee97d4452d7bb8464d8e658d04e1c3946dd60a5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 345095} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0a6fadf95f41075761c72e4cd5b854515c952d3e3dd60091017e10879662fa01', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 345117} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12607384, partition_space_available_1k=68483100, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1433148, cpu_share=0.04, mem_res=1361216, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13964, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55856, cpu_share=0.01, mem_res=43204, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7604872, free=262148, buffers=423892, cached=7893296, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.6, five_min_avg=2.33, fifteen_min_avg=2.49, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:05:57] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.002039 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:05:59] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.001318 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:06:04] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.001370 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '54659d04d10bd472c6366c35cdf8c70addd66f2397e7f31d1cf43ac13ec635aa', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 354946} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'da913970f372199ff28b9754a7483c1d3f9dcc68bf9708857e05d9be9e34548c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 354968} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12614336, partition_space_available_1k=68476148, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1434908, cpu_share=0.04, mem_res=1361576, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14060, cpu_share=0.0, mem_res=9308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55976, cpu_share=0.01, mem_res=43116, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7612344, free=267748, buffers=423456, cached=7880660, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.55, five_min_avg=2.45, fifteen_min_avg=2.52, cpu_share=11.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:06:16] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6755 0.001382 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '59e351859694f92774aa27bd89deb951d0ce77e95fd33f6557b54a95ac5e36c6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 357019} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e6a2ba63daf9caf82c1e0a4db0f58983a3384ec4f506ed31c17921684ef80644', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 357042} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12617052, partition_space_available_1k=68473432, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1441992, cpu_share=0.04, mem_res=1367492, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14208, cpu_share=0.0, mem_res=9440, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56072, cpu_share=0.01, mem_res=43096, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7642520, free=261764, buffers=422784, cached=7857140, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.06, five_min_avg=2.34, fifteen_min_avg=2.47, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0288b5047eec752248f4b45283d68e7a1636ed86c051acc7aeae2f59bc2918d3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 358107} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e64ed7791616c6ceeed8a7624822982ff1188961abfaa856343246ea543169d9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 358130} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12618964, partition_space_available_1k=68471520, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1437840, cpu_share=0.04, mem_res=1361568, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14216, cpu_share=0.0, mem_res=9440, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56204, cpu_share=0.01, mem_res=42984, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7630836, free=266720, buffers=423388, cached=7863264, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.01, five_min_avg=2.26, fifteen_min_avg=2.44, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '42204c0819c903f97d4412afa8a30ef200fe5a11ea6ddf82e70903fd5b096a95', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 359226} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a57931d01960fd881489199e4801f51490851bf214244828a877dec3354c93fe', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 359248} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12626336, partition_space_available_1k=68464148, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1441812, cpu_share=0.04, mem_res=1363896, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14144, cpu_share=0.0, mem_res=9572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56096, cpu_share=0.01, mem_res=42732, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7641228, free=272792, buffers=422748, cached=7847440, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.08, five_min_avg=2.23, fifteen_min_avg=2.41, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5c473e8006241cae71a8f6b64ae2705b780f9eefb9f190525ad588a90c6dd14d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 360326} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd8a551af0fa067d39eaef78b30a19f95f79b42c93d6a4dc11a0e0f293608cdc9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 360346} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12634008, partition_space_available_1k=68456476, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1445972, cpu_share=0.04, mem_res=1366012, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14156, cpu_share=0.0, mem_res=9572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56168, cpu_share=0.01, mem_res=42880, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7640944, free=260712, buffers=423300, cached=7859252, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.52, five_min_avg=2.04, fifteen_min_avg=2.33, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:10:48] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001452 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ada87e68ff5cce1f5c698e4266db8a47ae9a9c950e12a7f91ce98d41b8a86acf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 362045} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '73484e7db0e7a4af2c7ebc4ea9087117c55c6b36d925541dc24b385187932e29', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 362067} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12626284, partition_space_available_1k=68464200, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1449512, cpu_share=0.04, mem_res=1367944, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14220, cpu_share=0.0, mem_res=9572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56272, cpu_share=0.01, mem_res=42856, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7652308, free=249644, buffers=423364, cached=7858892, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.73, five_min_avg=2.0, fifteen_min_avg=2.3, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:11:47] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6751 0.001574 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:11:50] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6751 0.001953 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:11:54] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6751 0.001377 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4af206798be722b76f79192de905461f33540a32952024fd988b8e721219afd6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 372247} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b3e72fd27710d433356205fb07fccc99b8004fdf17943678f7d7ba7b6a1c30ab', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 372269} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12633028, partition_space_available_1k=68457456, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1452544, cpu_share=0.04, mem_res=1369892, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14244, cpu_share=0.0, mem_res=9572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56100, cpu_share=0.01, mem_res=42796, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7663528, free=264676, buffers=423980, cached=7832024, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.53, five_min_avg=2.22, fifteen_min_avg=2.36, cpu_share=11.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1245e74c2cfe5beab71ec58e053ab039676c6646846cd19e7c81a2949cbb860f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 373336} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a6a92395ba6d9ecba667990e625db84c561104efbccd2727468f3166642c3c31', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 373357} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12634924, partition_space_available_1k=68455560, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1455360, cpu_share=0.04, mem_res=1370824, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14260, cpu_share=0.0, mem_res=9572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56120, cpu_share=0.01, mem_res=42840, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7667048, free=257992, buffers=423996, cached=7835172, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.02, five_min_avg=2.14, fifteen_min_avg=2.33, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f01f8e135a00c3796ab72792b964ff0190aa4936f66cbc23b4ef1e96d902a2b2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 374449} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd49a223b007b1d81944811d8e818c666bb8272f8e2f36ad33d8e8700e6aff2ba', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 374473} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12636892, partition_space_available_1k=68453592, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1458512, cpu_share=0.04, mem_res=1372296, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14276, cpu_share=0.0, mem_res=9572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56332, cpu_share=0.01, mem_res=42772, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7670560, free=251152, buffers=424012, cached=7838484, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.94, five_min_avg=2.09, fifteen_min_avg=2.29, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '163d5f55cf6d148a1c7bd193ab627eab39e18c62afe00f8ca909da496aee9b91', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 375493} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9aaa81b6b6ec5c7c20bdc6dd388dc254aa62f59e92f54e39f51bc6e659096e50', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 375551} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12639152, partition_space_available_1k=68451332, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1462104, cpu_share=0.04, mem_res=1373840, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14220, cpu_share=0.0, mem_res=9572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56328, cpu_share=0.01, mem_res=42880, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7674740, free=243576, buffers=424024, cached=7841868, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.91, five_min_avg=2.06, fifteen_min_avg=2.27, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:15:50] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6754 0.001400 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4d0976ddc2f2187fda6e298a67db7ec7696d0cbadac78056499c6f59b5d9489b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 377084} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd8a1f80c3babeb604711d50c9daeabe3b663c38182a2f8233f7e2e78c9e97b38', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 377192} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12646532, partition_space_available_1k=68443952, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1464568, cpu_share=0.04, mem_res=1375276, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14276, cpu_share=0.0, mem_res=9704, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56144, cpu_share=0.01, mem_res=42796, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7682352, free=252864, buffers=423352, cached=7825640, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.82, five_min_avg=2.01, fifteen_min_avg=2.24, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0045781d30f1be09e0d5f2fe961b280f2073b608c57ae3d956fc4685e23db250', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 379530} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '765b0082c96076cf19a676470b7fd365eee51748123eb198913147c93e144ab7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 379620} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12649168, partition_space_available_1k=68441316, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1467260, cpu_share=0.04, mem_res=1377096, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14280, cpu_share=0.0, mem_res=9704, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56244, cpu_share=0.01, mem_res=42824, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7723600, free=282436, buffers=419712, cached=7758460, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.56, five_min_avg=2.12, fifteen_min_avg=2.25, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:17:30] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001463 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:17:34] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001483 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:17:38] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001990 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '17477e959bd5ef12fc882c063ea200c8cb7389d66bb3c0ab5e617962ec20c984', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 388189} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '032d5f273706db1754849e456baa005330021ba3901b12e73b97ed81e66fba43', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 388212} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12655332, partition_space_available_1k=68435152, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1470364, cpu_share=0.04, mem_res=1378788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14340, cpu_share=0.0, mem_res=9836, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56268, cpu_share=0.01, mem_res=42868, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7695684, free=309940, buffers=422000, cached=7756584, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.67, five_min_avg=2.46, fifteen_min_avg=2.36, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ede0711471939756592f3798a633aecec5612eb1956318d87b9eb68d877ea565', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 389293} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '15c6d0efbe285314d205dfa06603c2c8888a0b086028e42d8396e102ba854929', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 389315} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12665540, partition_space_available_1k=68424944, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1472844, cpu_share=0.04, mem_res=1379480, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14344, cpu_share=0.0, mem_res=9836, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56220, cpu_share=0.01, mem_res=42832, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7699792, free=302224, buffers=422008, cached=7760184, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.34, five_min_avg=2.28, fifteen_min_avg=2.31, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5fec9379f8cbfdead401127ecea6c0155799c49ff9376e7582aad77bce053abd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 390379} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd20e6447cea062cd221267e8c9b6d6f26e21829fcd7b94d6f196971c2dd48102', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 390400} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12668224, partition_space_available_1k=68422260, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1476420, cpu_share=0.04, mem_res=1381384, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14352, cpu_share=0.0, mem_res=9836, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56256, cpu_share=0.01, mem_res=43060, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7721956, free=276284, buffers=422040, cached=7763928, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.64, five_min_avg=2.08, fifteen_min_avg=2.23, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:20:20] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6754 0.001637 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '593290fe95a21b0e2ac56d072ef858c0cbb14b2bab11a2fc4c152ba1429de556', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 392500} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b764c5cf134e0ee187e45ec5b0792130ea47f0189f2d533846beab3e4e12f16e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 392522} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12670760, partition_space_available_1k=68419724, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1479920, cpu_share=0.04, mem_res=1383344, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14292, cpu_share=0.0, mem_res=9676, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56256, cpu_share=0.01, mem_res=43004, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7664896, free=328592, buffers=422044, cached=7768676, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.43, five_min_avg=2.27, fifteen_min_avg=2.29, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c9b66a4d47ecefe8fe7729593942f439bbd8e7c4b8944649cd03d707663750cc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 393565} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f2688ec256efcfc1ad5b3fdcba5ae2bf3004f7262b571537266ba498703bd46f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 393587} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12677924, partition_space_available_1k=68412560, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1482384, cpu_share=0.04, mem_res=1383980, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14292, cpu_share=0.0, mem_res=9676, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56276, cpu_share=0.01, mem_res=43024, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7663220, free=322012, buffers=422188, cached=7776788, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.66, five_min_avg=2.29, fifteen_min_avg=2.29, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:22:55] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001444 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:22:58] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001412 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:23:02] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001228 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cdbfd09835f10e2517ce02ced5d4464245f5d1c8525130a1e6a8226cc23ebc5c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 403461} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2cc7776ebd3bbf590e5ef6d6dbdcf67547aa936595b65b757ad2c16a5fe8e957', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 403482} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12684860, partition_space_available_1k=68405624, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1484212, cpu_share=0.04, mem_res=1384128, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14340, cpu_share=0.0, mem_res=9676, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56392, cpu_share=0.01, mem_res=42980, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7670172, free=299184, buffers=422880, cached=7791972, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.72, five_min_avg=2.38, fifteen_min_avg=2.32, cpu_share=11.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7a544d81a21bd818470adec99a6a5290f22ac1565bc5ffaaa01e953f07de1669', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 404580} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9a6205c9873c5cdfd9b4736a9b45d7d9f05436059e5a6b43c4b0d57b396c4717', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 404601} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12682120, partition_space_available_1k=68408364, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1487376, cpu_share=0.04, mem_res=1385548, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14344, cpu_share=0.0, mem_res=9676, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56420, cpu_share=0.01, mem_res=43056, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7679088, free=291916, buffers=422884, cached=7790320, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.23, five_min_avg=2.3, fifteen_min_avg=2.3, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:24:49] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6752 0.001570 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '57d36f74093d4182b6650c4e787ce81d8f27570d399df80040d6a6f21065de42', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 406328} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '23c7c6886a2811cfb46e12beb63a56f57d5fe2891fb7d73c3b4f0d062b4dd481', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 406350} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12685544, partition_space_available_1k=68404940, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1491356, cpu_share=0.04, mem_res=1387604, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14344, cpu_share=0.0, mem_res=9676, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56228, cpu_share=0.01, mem_res=43064, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7676588, free=289624, buffers=422920, cached=7795076, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.62, five_min_avg=2.12, fifteen_min_avg=2.23, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '249ce51866c234d958f1023aa0e34695f7ba9c4e23c30e37b896948af706f249', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 407432} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '46a39e6c8ab095d748fefe4ae623d0f068d270834b5bbaae01db70f96f3ecef2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 407454} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12687428, partition_space_available_1k=68403056, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1494912, cpu_share=0.04, mem_res=1389704, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14352, cpu_share=0.0, mem_res=9676, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56456, cpu_share=0.01, mem_res=43240, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7678732, free=284528, buffers=422944, cached=7798004, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.33, five_min_avg=1.95, fifteen_min_avg=2.16, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a962182bc7c847b930c8335bf76866af8aa9e9781846ada1c80576b726565885', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.50', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 414713} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:27:09] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.001497 CRITICAL:root:Exit data: {'ID': 'c5bee71579d1283051a647adadd8986de9ab2c9de59e1e76a4bdde8126e52f92', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '6962f81038654cb7ecf9b94e25f70c30b0617f16b7871e51bd343c3fe09ea5a5', 'DetachKeys': '', 'Pid': 414736} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.50 - - [2025-08-28 02:27:12] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6753 0.002280 DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-k8s-1433-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=12696884, partition_space_available_1k=68393600, percentage_partition_space_used=16, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1498460, cpu_share=0.04, mem_res=1391536, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14340, cpu_share=0.0, mem_res=9676, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56456, cpu_share=0.01, mem_res=43164, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16184208, used=7736412, free=212044, buffers=422368, cached=7813384, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.75, five_min_avg=1.92, fifteen_min_avg=2.14, cpu_share=11.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo')