++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ mkdir -p /var/log/contrail/database-nodemgr ++ log_file=/var/log/contrail/database-nodemgr/console.log ++ touch /var/log/contrail/database-nodemgr/console.log ++ chmod 600 /var/log/contrail/database-nodemgr/console.log ++ exec +++ tee -a /var/log/contrail/database-nodemgr/console.log +++ date ++ echo 'INFO: =================== Thu Aug 28 02:08:15 AM UTC 2025 ===================' INFO: =================== Thu Aug 28 02:08:15 AM UTC 2025 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ awk '{print $2}' +++++ grep -o 'dev.*' +++++ ip route get 1 ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ cut -d / -f 1 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ ip addr show dev ens3 +++ head -n 1 +++ awk '{print $2}' +++ grep 'inet ' ++ DEFAULT_LOCAL_IP=10.0.0.38 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=256 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 256 ]] ++ [[ -z '' ]] +++ tail -n 1 +++ mount -t hugetlbfs +++ awk '/pagesize=2M/{print($3)}' ++ HUGE_PAGES_2MB_DIR= ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=openstack ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=rbac ++ AUTH_MODE=keystone ++ AUTH_PARAMS= ++ SSL_ENABLE=false ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_ALARM_ENABLE=True ++ ANALYTICS_SNMP_ENABLE=True ++ ANALYTICSDB_ENABLE=True ++ ANALYTICS_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICSDB_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_SNMP_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8081 ' +++ '[' -n '10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081 ' ']' +++ echo '10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081' ++ ANALYTICS_SERVERS='10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081' +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9042 ' +++ '[' -n '10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042 ' ']' +++ echo '10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042' ++ ANALYTICSDB_CQL_SERVERS='10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042' ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8086 ' +++ '[' -n '10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086 ' ']' +++ echo '10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086' ++ COLLECTOR_SERVERS='10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086' ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ CONFIGDB_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8082 ' +++ '[' -n '10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082 ' ']' +++ echo '10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082' ++ CONFIG_SERVERS='10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082' +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9161 ' +++ '[' -n '10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161 ' ']' +++ echo '10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161' ++ CONFIGDB_SERVERS='10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161' +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9041 ' +++ '[' -n '10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041 ' ']' +++ echo '10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041' ++ CONFIGDB_CQL_SERVERS='10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041' ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=false ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=false ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=false ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.20.0.25,10.20.0.17,10.20.0.254 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.20.0.25,10.20.0.17,10.20.0.254 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+=10.0.0.19:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+=10.0.0.38:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+=10.0.0.41:2181, +++ '[' -n 10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181, ']' +++ echo 10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181 ++ ZOOKEEPER_SERVERS=10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:2181 ' +++ '[' -n '10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181 ' ']' +++ echo '10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181' ++ ZOOKEEPER_SERVERS_SPACE_DELIM='10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181' ++ RABBITMQ_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+=10.0.0.19:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+=10.0.0.38:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+=10.0.0.41:5673, +++ '[' -n 10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673, ']' +++ echo 10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673 ++ RABBITMQ_SERVERS=10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=false ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:6379 ' +++ '[' -n '10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379 ' ']' +++ echo '10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379' ++ REDIS_SERVERS='10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379' ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=false ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9092 ' +++ '[' -n '10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092 ' ']' +++ echo '10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092' ++ KAFKA_SERVERS='10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092' ++ KAFKA_SSL_ENABLE=false ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=10.0.0.19 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=false ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=false ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=false ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ /v3 == \/\v\2\.\0 ]] ++ [[ openstack == \o\p\e\n\s\t\a\c\k ]] ++ AUTH_MODE=keystone ++ [[ keystone == \k\e\y\s\t\o\n\e ]] ++ AUTH_PARAMS='--admin_password contrail123' ++ AUTH_PARAMS+=' --admin_tenant_name admin' ++ AUTH_PARAMS+=' --admin_user admin' ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ keystone == \k\e\y\s\t\o\n\e ]] + cat + [[ http == \h\t\t\p\s ]] + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + [[ database == \t\o\r\a\g\e\n\t ]] + export NODEMGR_TYPE=contrail-database + NODEMGR_TYPE=contrail-database + NODEMGR_NAME=contrail-database-nodemgr ++ echo DATABASE ++ tr - _ + ntype=DATABASE + [[ DATABASE == \V\R\O\U\T\E\R ]] + [[ DATABASE == \D\A\T\A\B\A\S\E ]] + htype=ANALYTICSDB + [[ DATABASE == \C\O\N\T\R\O\L ]] ++ get_listen_ip_for_node ANALYTICSDB +++ cut -d ' ' -f 1 +++ find_my_ip_and_order_for_node ANALYTICSDB +++ local server_typ=ANALYTICSDB_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.19,10.0.0.38,10.0.0.41 +++ local servers=10.0.0.19,10.0.0.38,10.0.0.41 +++ local server_list= +++ IFS=, +++ read -ra server_list ++++ tr '\n' , ++++ get_local_ips ++++ cat /proc/net/fib_trie ++++ uniq ++++ sort ++++ grep -vi host ++++ awk '/32 host/ { print f } {f=$2}' +++ local local_ips=,10.0.0.38,10.20.0.17,127.0.0.1,172.17.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.19'\''))' +++ local server_ip=10.0.0.19 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.19 ]] +++ [[ ,10.0.0.38,10.20.0.17,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.19, ]] +++ (( ord+=1 )) +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.38'\''))' +++ local server_ip=10.0.0.38 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.38 ]] +++ [[ ,10.0.0.38,10.20.0.17,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.38, ]] +++ echo 10.0.0.38 2 +++ return ++ local ip=10.0.0.38 ++ [[ -z 10.0.0.38 ]] ++ echo 10.0.0.38 + hostip=10.0.0.38 + introspect_ip=0.0.0.0 + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + mkdir -p /etc/contrail + cat ++ format_boolean false ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ echo False + '[' -n '' ']' + cat + add_ini_params_from_env DATABASE_NODEMGR /etc/contrail/contrail-database-nodemgr.conf + local service_name=DATABASE_NODEMGR + local cfg_path=/etc/contrail/contrail-database-nodemgr.conf + local delim=__ ++ sed 's/^DATABASE_NODEMGR__//g' ++ cut -d = -f 1 ++ sort ++ grep '^DATABASE_NODEMGR__.*__.*=.*$' ++ set -o posix ++ set + local vars=DEFAULTS__minimum_diskGB + local section= + for var in $vars + local var_name=DATABASE_NODEMGR__DEFAULTS__minimum_diskGB + local val=2 ++ echo DEFAULTS__minimum_diskGB ++ sed s/DASH/-/g + local var_unsubst=DEFAULTS__minimum_diskGB ++ echo DEFAULTS__minimum_diskGB ++ sed 's/^\(.*\)__.*$/\1/' + local var_section=DEFAULTS + [[ '' != \D\E\F\A\U\L\T\S ]] + echo '[DEFAULTS]' + section=DEFAULTS ++ echo DEFAULTS__minimum_diskGB ++ sed 's/.*__\(.*\)$/\1/' + local var_param=minimum_diskGB + echo 'minimum_diskGB = 2' + cat /etc/contrail/contrail-database-nodemgr.conf [DEFAULTS] http_server_ip=0.0.0.0 log_file=/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log log_level=SYS_DEBUG log_local=1 hostip=10.0.0.38 db_port=9042 db_jmx_port=7200 db_use_ssl=False [COLLECTOR] server_list=10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086 [SANDESH] introspect_ssl_enable=false sandesh_ssl_enable=false [DEFAULTS] minimum_diskGB = 2 + upgrade_old_logs contrail-database-nodemgr + local template=contrail-database-nodemgr ++ dirname /var/log/contrail/database-nodemgr + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/database-nodemgr + chmod 755 /var/log/contrail/database-nodemgr + mv -n '/var/log/contrail/*contrail-database-nodemgr.log*' /var/log/contrail/database-nodemgr/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/database-nodemgr + find /var/log/contrail/database-nodemgr -uid 0 -exec chown 1999:1999 '{}' + + exec /bin/sh -c '/usr/local/bin/contrail-nodemgr --nodetype=${NODEMGR_TYPE}' 08/28/2025 02:08:16.424 7ff1a302f040 [contrail-database-nodemgr] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 08/28/2025 02:08:16.522 7ff1a302f040 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 02:08:16.530 7ff1a302f040 [contrail-database-nodemgr] [INFO]: SANDESH: INTROSPECT IS ON: 0.0.0.0:8103 08/28/2025 02:08:16.534 7ff1a302f040 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 02:08:16.540 7ff1a302f040 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 08/28/2025 02:08:16.540 7ff1a302f040 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log] Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: CRITICAL:root:Exit data: {'ID': 'b22dc93bd8fbf39a4d8896c44cffa31f0fdcbe848c19fe60db14fb96e965d607', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 29677} ERROR:contrail-database-nodemgr:Failed to connect to database by CQL and socket: Result: 1 INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-2.', _node_type='Database', _instance_id='0', _timestamp=1756346897204464, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Non-Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=[''], status='Down', description='none to Idle on EvStart')]), flag_infos=None, description='Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': 'f0f210cf7093ea837e3783a944ce968cedae4c93e1b092358328eadc8beef5db', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 29705} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.38:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:send_process_state_db: Sending UVE: NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-2.', _node_type='Database', _instance_id='0', _timestamp=1756346903438354, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=False, status=None, process_status=None, process_info=[ProcessInfo(process_name='cassandra', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-query-engine', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-database-nodemgr', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[])], disk_usage_info=None, description=None, all_core_file_list=None, build_info='{"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-08-28 00:14:38.756636", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10583-1.novalocal", "build-id" : "v0+nightly", "build-number" : "nightly.el9"}]}', process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=7006416, partition_space_available_1k=74084068, percentage_partition_space_used=9, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=[], build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1262396, cpu_share=0.0, mem_res=1254168, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=7632, cpu_share=0.0, mem_res=5836, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54644, cpu_share=0.0, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=4361516, free=4711896, buffers=219424, cached=6882796, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.51, five_min_avg=1.33, fifteen_min_avg=0.55, cpu_share=0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. deleted = False process_info = [ << process_name = cassandra process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-query-engine process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-database-nodemgr process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, ] build_info = {"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-08-28 00:14:38.756636", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10583-1.novalocal", "build-id" : "v0+nightly", "build-number" : "nightly.el9"}]} >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. disk_usage_info = { /dev/vda1 : << partition_type = ext4 partition_space_used_1k = 7006416 partition_space_available_1k = 74084068 percentage_partition_space_used = 9 >> } all_core_file_list = [ ] process_mem_cpu_usage = { cassandra : << mem_virt = 1262396 cpu_share = 0.0 mem_res = 1254168 >>contrail-query-engine : << mem_virt = 7632 cpu_share = 0.0 mem_res = 5836 >>contrail-database-nodemgr : << mem_virt = 54644 cpu_share = 0.0 mem_res = 45132 >> } system_mem_usage = << total = 16175632 used = 4361516 free = 4711896 buffers = 219424 cached = 6882796 node_type = database-node >> system_cpu_usage = << one_min_avg = 2.51 five_min_avg = 1.33 fifteen_min_avg = 0.55 cpu_share = 0 node_type = database-node >> >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.38:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.38:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-database-nodemgr:Session Event: TCP Connected INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] ERROR:contrail-database-nodemgr:Session Event: TCP Connection Closed INFO:contrail-database-nodemgr:Processing event[EvTcpClose] in state[ClientInit] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpClose] => State[ClientInit] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = ClientInit to Idle on EvTcpClose >>, ] description = Cassandra state detected DOWN. Collector:Collector[ClientInit to Idle on EvTcpClose] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 1 pid = 1 http_port = 8103 start_time = 1756346896552582 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.38:8086, 10.0.0.19:8086, 10.0.0.41:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-database-nodemgr:Session Event: TCP Connected INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-2. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001017 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[ClientInit] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 0 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Client: Event[EvSandeshCtrlMessageRecv] => State[ClientInit] -> State[Established] INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001438 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[Established] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 3 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '63254a7bd8a20a03a8f4b9f1126605d4e351e8e38ae90f74e2290c57871a4ff1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 34852} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-2.', _node_type='Database', _instance_id='0', _timestamp=1756346957509940, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=['10.0.0.38:8086'], status='Up', description='ClientInit to Established on EvSandeshCtrlMessageRecv')]), flag_infos=None, description='')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) CRITICAL:root:Exit data: {'ID': '455bb142408522f4d587b4a00fc3efc8aa0df9ffd86168e361995d880f1f34ec', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 34872} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8332592, partition_space_available_1k=72757892, percentage_partition_space_used=10, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1364684, cpu_share=0.38, mem_res=1346868, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=11652, cpu_share=0.0, mem_res=9672, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54620, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=5529248, free=2330928, buffers=265644, cached=8049812, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.97, five_min_avg=1.62, fifteen_min_avg=0.69, cpu_share=9.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '879fedd508c8cd50283ecf5d54752a43c3660dcf5253c91f643c22ed42d898bb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 36858} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f281e5ab7e8f439484bf841421aea71aeb5bba1c13b9054b8dc9544c621f28eb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 36880} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8362976, partition_space_available_1k=72727508, percentage_partition_space_used=10, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1370432, cpu_share=0.08, mem_res=1349200, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=11896, cpu_share=0.0, mem_res=9916, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54628, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=5944228, free=1886812, buffers=267740, cached=8076852, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.21, five_min_avg=1.74, fifteen_min_avg=0.8, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0c0958aee002f69220f39aa68eb6e221292fb063247bb32f5d32bd9b18ab4baa', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 37462} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '195f3c7ded057c47d74648450d56f8e4e726145aa89d05975f2f420f24923046', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 37484} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8368468, partition_space_available_1k=72722016, percentage_partition_space_used=10, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1370584, cpu_share=0.09, mem_res=1347192, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12108, cpu_share=0.0, mem_res=10128, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54628, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=5950240, free=1873092, buffers=267844, cached=8084456, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.46, five_min_avg=1.63, fifteen_min_avg=0.83, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.38 - - [2025-08-28 02:11:34] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6757 0.001898 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.38 - - [2025-08-28 02:11:50] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6757 0.001452 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2d250e18f0104a39d961195f0d9e0b492882bcd6488489dda28bb817ac4b1843', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 38560} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '68ca815e491dbbc65d6d7203584020a09312f2e9a8d174e409f452e18114cb99', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 38583} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8540440, partition_space_available_1k=72550044, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1375024, cpu_share=0.07, mem_res=1348788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12304, cpu_share=0.0, mem_res=10324, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54636, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=5996152, free=1641164, buffers=273132, cached=8265184, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.09, five_min_avg=1.5, fifteen_min_avg=0.83, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 02:13:08] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6756 0.001975 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7bef3f9465d2752b4064064de93151d9010c0fa040345a1ddb7ef58e7a212f2f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 39552} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '66eb66e3e56757373c03c709599810c8a4eabb8b995ec81bc1ef8f21c4cbc722', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 39573} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8546524, partition_space_available_1k=72543960, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1379360, cpu_share=0.08, mem_res=1349508, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12444, cpu_share=0.0, mem_res=10464, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54656, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6042744, free=1587720, buffers=273208, cached=8271960, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.08, five_min_avg=1.41, fifteen_min_avg=0.84, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cf2648a756385cbd0a6363f552a854a6fefe263930867af92571a4c968874476', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 40303} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '95680176dcbc9af35d0e75d97f66b52f55ef32c394199df576d4bd1f3ee86235', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 40324} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8560224, partition_space_available_1k=72530260, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1399456, cpu_share=0.18, mem_res=1361356, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14116, cpu_share=0.0, mem_res=12104, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54664, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6097444, free=1517792, buffers=273316, cached=8287080, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.53, five_min_avg=1.47, fifteen_min_avg=0.9, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8bc1496d18df0f13595568ada66dc853d6f69a72464c2567073e5ae3df7426b3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 40927} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0fbd666330ef32c905dd1350c291de3d5003e60e58961659010364e220478276', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 40953} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8568776, partition_space_available_1k=72521708, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1404828, cpu_share=0.07, mem_res=1362240, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13532, cpu_share=0.0, mem_res=11524, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54676, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6107012, free=1500868, buffers=273368, cached=8294384, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.12, five_min_avg=1.37, fifteen_min_avg=0.91, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e58c963e1d5032d7d51064d3f89ecc3a8eccf6dc7add9c24eb72b710671a2a8b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 41582} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8cf455709746779eaa6e3831ebff5dc93252398650a04ab704b9e4b9dd6f95e9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 41609} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8580128, partition_space_available_1k=72510356, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1410364, cpu_share=0.12, mem_res=1362388, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13636, cpu_share=0.0, mem_res=11628, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54676, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6131984, free=1461524, buffers=273464, cached=8308660, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.27, five_min_avg=1.36, fifteen_min_avg=0.93, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '369659d5c2bd8847b39f48f317fb71810e843bcdec003e55740e39eb46445e59', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 42297} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5ef98bb7a3cfcca65f2e56855ab168bd3554bd695e71f92a27c124925ca10494', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 42319} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8597804, partition_space_available_1k=72492680, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1424356, cpu_share=0.13, mem_res=1367784, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13704, cpu_share=0.0, mem_res=11696, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54684, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6160180, free=1416748, buffers=273572, cached=8325132, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.48, five_min_avg=1.74, fifteen_min_avg=1.09, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6e1472cf511be6aafba8d68690ddf1393ce8eab50df9f107f86cce0dd1b801c5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 42854} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '76e2966632331723f7b06380e04e6733b25d5cc160a064aac68871d2813ffb23', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 42879} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8603480, partition_space_available_1k=72487004, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1431700, cpu_share=0.06, mem_res=1371712, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13752, cpu_share=0.0, mem_res=11744, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54684, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6169596, free=1401296, buffers=273668, cached=8331072, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.68, five_min_avg=1.66, fifteen_min_avg=1.1, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ef86769f37ae62364acd71f43e0f7fee9d2ee2938916012db68aca9d33696c65', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 43422} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b3f287319eb3cf9ab5a6f4d370eb6041ef464570d40b7cbd0c980ab78b790bd5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 43444} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8609384, partition_space_available_1k=72481100, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1439908, cpu_share=0.06, mem_res=1376008, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13808, cpu_share=0.0, mem_res=11800, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54688, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6183156, free=1381324, buffers=273752, cached=8337400, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.02, five_min_avg=1.47, fifteen_min_avg=1.07, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8f1d6fa011e74e106fa1cfbd3dc6c137f6aba324a8386402f9895c13348a017b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 44173} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1feb50346cde9b69ce5f3eef83c686871e82e1cb29da02aaae3353dee82462d5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 44196} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8619020, partition_space_available_1k=72471464, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1456940, cpu_share=0.08, mem_res=1387736, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13904, cpu_share=0.0, mem_res=11896, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54692, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6200996, free=1352264, buffers=273816, cached=8348556, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.18, five_min_avg=1.75, fifteen_min_avg=1.19, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7a54ccdacc210aa494af610ee779915de06603f007e3d96bc0173d2e3db3f80f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 44830} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '84aa78f132deae65f5ac5dd50166b968c58c5e0f01da2459f0d2d663076fab1c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 44850} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8632100, partition_space_available_1k=72458384, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1464140, cpu_share=0.11, mem_res=1388540, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13852, cpu_share=0.0, mem_res=11844, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54700, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6205900, free=1334208, buffers=273912, cached=8361612, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.7, five_min_avg=2.05, fifteen_min_avg=1.33, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '040138adab71d1e7058e24c4f578475dcdae13d39fa8991c6452b166bde04d31', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 45420} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9e325600edf7f76e5da361d2c18643915499d2720d2e4ac68a5920b9604ecb22', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 45442} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8646100, partition_space_available_1k=72444384, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1479656, cpu_share=0.1, mem_res=1396396, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13908, cpu_share=0.0, mem_res=11900, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54708, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6226356, free=1299380, buffers=274032, cached=8375864, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.64, five_min_avg=1.85, fifteen_min_avg=1.31, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e1be0a30edb77758943c7414a33d01bbb0e1927aea788a68ed100269a73b5478', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 46010} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '994a0ffba9e6b383dad4016d2ee19f4b55912619ebe49c130fbe648068c24418', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 46031} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8654516, partition_space_available_1k=72435968, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1489384, cpu_share=0.07, mem_res=1401416, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13924, cpu_share=0.0, mem_res=11916, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54708, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6234132, free=1283472, buffers=274108, cached=8383920, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.3, five_min_avg=1.72, fifteen_min_avg=1.3, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8ddd1c9389a53488c3c06710ca0ec5b5968a8004dcb2e2a96254cd5f2b078b84', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 46616} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '638ea1613412eb9449afa9d9f48944dc4adf624386f01ab1c1e6d54f81964541', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 46637} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8665188, partition_space_available_1k=72425296, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1501028, cpu_share=0.09, mem_res=1407928, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13992, cpu_share=0.0, mem_res=11984, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54720, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6248264, free=1258420, buffers=274196, cached=8394752, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.39, five_min_avg=1.66, fifteen_min_avg=1.3, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '040db0654d50fd1f733d5a5f25677b0274bba73f8eb844cd475a511003935d83', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 47183} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c3e36a3f59c49acbd5f477b7f9be967e9c81a201add88c0088063e64bd636925', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 47203} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8671496, partition_space_available_1k=72418988, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1508484, cpu_share=0.06, mem_res=1411404, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13992, cpu_share=0.0, mem_res=11984, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54720, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6252992, free=1247168, buffers=274288, cached=8401184, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.18, five_min_avg=1.54, fifteen_min_avg=1.28, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '47d0836c0db71111a776a0932df5768c3cbac8e03e04bf59eb7a951c5532b34b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 47737} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '765d70a8c317bdadaa59dfc24db8f1212fb46567abb7215f14fc5e9ee1ab792f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 47758} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8677460, partition_space_available_1k=72413024, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1516516, cpu_share=0.06, mem_res=1415536, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14036, cpu_share=0.0, mem_res=12028, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54720, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6259988, free=1233588, buffers=274360, cached=8407696, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.13, five_min_avg=1.46, fifteen_min_avg=1.27, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2b5ac77ca0ed241422b142cd1c24e5e5f4d3fa77ffdfbf47cd1db0086040d24b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 48297} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e9719c503cf70b81a5e380fc758a8c5af7ec0e3a2d97f5064214540ced1314c9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 48317} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8687452, partition_space_available_1k=72403032, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1527976, cpu_share=0.07, mem_res=1421316, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13992, cpu_share=0.0, mem_res=11984, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54732, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6272536, free=1210800, buffers=274432, cached=8417864, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.47, five_min_avg=1.47, fifteen_min_avg=1.28, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '27dc291ccd4fcf4b106b1bc78100aaf41727322ba5282272e19b6e250290a414', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 48910} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7c9675e064c2b96a04851447ecfb2def8edb7eaaee7453c9a5f9dd786c063cd5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 48931} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8694840, partition_space_available_1k=72395644, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1536576, cpu_share=0.06, mem_res=1425896, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14004, cpu_share=0.0, mem_res=11996, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54732, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6275788, free=1200116, buffers=274512, cached=8425216, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.14, five_min_avg=1.4, fifteen_min_avg=1.27, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '94183d23db1103961adac68e4062c06c1634ec9f6194be18d495d7601779eac0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 49546} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e35add293667506909716eee5153a4253b3307b8e3240b1f39b738c9e6eb4124', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 49568} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8728168, partition_space_available_1k=72362316, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1577664, cpu_share=0.17, mem_res=1436356, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14020, cpu_share=0.0, mem_res=12012, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54740, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6291444, free=1150040, buffers=274592, cached=8459556, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.08, five_min_avg=1.32, fifteen_min_avg=1.25, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4d0ac65859da06581450781302896cfb42c7b58c5e517ec3fcf42d1e9e9267b1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 50108} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b4ee5830fc1cb51269021612fba19f6b001fdb3df284f4d9ea0ed039bf277c84', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 50131} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8733940, partition_space_available_1k=72356544, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1579684, cpu_share=0.06, mem_res=1435512, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14076, cpu_share=0.0, mem_res=12068, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54740, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6294320, free=1140900, buffers=274644, cached=8465768, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.03, five_min_avg=1.26, fifteen_min_avg=1.23, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6b3841c149750e81787f41b7e1ca22aafc9e3f3016b6158784bbf39c70d031b6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 50769} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '673d888a2802ea23f627931af5ce9dbf15679effde6da92157786c761178717e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 50791} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8745760, partition_space_available_1k=72344724, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1586836, cpu_share=0.1, mem_res=1435788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14132, cpu_share=0.0, mem_res=12124, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54744, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6296152, free=1125808, buffers=274724, cached=8478948, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.41, five_min_avg=1.89, fifteen_min_avg=1.45, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '87df5c81ef8b19c980f5cacd27a9f599aa6cb57e1fb8dd13b8699d3fed5c53c8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 51322} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1f4fe95d1eceddeb15639e606766f8d31a7abb684fe3d816c34abe49d2fa82ca', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 51343} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8752380, partition_space_available_1k=72338104, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1590084, cpu_share=0.06, mem_res=1435588, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14140, cpu_share=0.0, mem_res=12132, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54748, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6296532, free=1119552, buffers=274760, cached=8484788, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.83, five_min_avg=1.71, fifteen_min_avg=1.41, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'fca85e4a9ed2a0287a43e006ffb63223517e893c696fccbeef76b8743aa79daf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 51879} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3d38d614329af12193143fe9ba4e5c419f6af4bac8a3b20ffbb9e4d93f96b49c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 51901} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8752968, partition_space_available_1k=72337516, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1593600, cpu_share=0.06, mem_res=1435792, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14100, cpu_share=0.0, mem_res=12092, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54756, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6294060, free=1121348, buffers=274804, cached=8485420, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.34, five_min_avg=1.6, fifteen_min_avg=1.39, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ea48e53b42014bcee6810f2a64fa327e9490a0ac7d77cf52d32cc1ca36d256d6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 52491} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '890e009bb7dc5abeff98662601788f7ea43cd15e2b7b307c90a8d594971c9bc9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 52513} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8758604, partition_space_available_1k=72331880, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1601868, cpu_share=0.09, mem_res=1438312, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14124, cpu_share=0.0, mem_res=12116, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54760, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6307332, free=1102212, buffers=274848, cached=8491240, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.99, five_min_avg=1.72, fifteen_min_avg=1.45, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b14e0e5966a5b5bf2787a74692a906d7662318cd7fc217b6f4cef3dff84a1947', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 53048} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '497089638ee01abba56df00a3f8daa7ae39843b14ce24c1d28663631b1e3d99f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 53072} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8765028, partition_space_available_1k=72325456, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1606268, cpu_share=0.06, mem_res=1438256, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14192, cpu_share=0.0, mem_res=12184, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54764, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6312044, free=1090700, buffers=274908, cached=8497980, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.53, five_min_avg=1.65, fifteen_min_avg=1.44, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c53f1833b98eb6d4c8f8d068923c52af9ea12da1d065e189043177ebf1d2dc85', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 53636} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e2df8974ca955a3a4f8dd2159730010063f01ef27e2cc6cb1dd296e2f5f6abfd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 53661} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8772516, partition_space_available_1k=72317968, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1612988, cpu_share=0.07, mem_res=1440152, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14252, cpu_share=0.0, mem_res=12240, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54768, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6286516, free=1112344, buffers=274952, cached=8501820, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.87, five_min_avg=1.44, fifteen_min_avg=1.38, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6f6f5aea7af2c38f5167e69cfb0a9b29e54d1f74634d97a0c7b29d0109ca0101', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 54240} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '10d7e075b5263c6aadb77cbcec7cee485c241743cdcc85d9327368810228d2dd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 54262} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8787284, partition_space_available_1k=72303200, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1619920, cpu_share=0.08, mem_res=1430912, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14252, cpu_share=0.0, mem_res=12240, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54772, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6295092, free=1084584, buffers=275060, cached=8520896, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.95, five_min_avg=1.34, fifteen_min_avg=1.35, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ac77b50bca2f665acce0afb150a71d86e9314cffb073be88a785d756fe7b85b5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 54847} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7f332fde5be820a745fd6b5feabac1faf7d46b464d08b75ab24e0cec71c49796', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 54869} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8797172, partition_space_available_1k=72293312, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1623816, cpu_share=0.08, mem_res=1430216, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14252, cpu_share=0.0, mem_res=12240, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54780, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6305492, free=1064224, buffers=275144, cached=8530772, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.83, five_min_avg=1.49, fifteen_min_avg=1.39, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a5c8073d32af5079b09670acdffb8ae0a53d20c90e85bd56fda9811ca02c7218', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 55410} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3d20a24db6d7701dca49e8d08fa8c4b780faaed656e50f50f1b1d878893c0761', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 55432} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8803200, partition_space_available_1k=72287284, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1627752, cpu_share=0.06, mem_res=1430652, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14228, cpu_share=0.0, mem_res=12216, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54780, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6306680, free=1061776, buffers=275216, cached=8531960, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.09, five_min_avg=1.34, fifteen_min_avg=1.35, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f1f182598d80966f71cf29cdbf9f9ded923677d4ee54a6d4826f5724452e2b21', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 55970} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5f2b273305685203aeb993a892c50d07759d696ac5364192a2f08471c89e9732', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 55989} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8802256, partition_space_available_1k=72288228, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1632792, cpu_share=0.07, mem_res=1430464, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14244, cpu_share=0.0, mem_res=12232, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54788, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6311108, free=1052236, buffers=275300, cached=8536988, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.81, five_min_avg=1.46, fifteen_min_avg=1.39, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bdb09e549ee67402cf682c915866248e467a3480031608bf10e56a583af21d06', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 56562} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd780b0560cb4c878ebe583e8cf20a36d45d96945a19a50e389c6940f0a4a3ec8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 56583} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8812096, partition_space_available_1k=72278388, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1638412, cpu_share=0.08, mem_res=1430752, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14244, cpu_share=0.0, mem_res=12232, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54792, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6308084, free=1045324, buffers=275372, cached=8546852, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.23, five_min_avg=1.37, fifteen_min_avg=1.36, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '615a003f6ccc9db0bf3ca15dbd61d3aab03d88815379d53a0279e35d31667133', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 57121} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '10b2388e153aec311d62d60ccbe4eda8cb56d7c2c8b5fb5fb87cca4776f86e9f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 57142} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8818260, partition_space_available_1k=72272224, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1641644, cpu_share=0.05, mem_res=1430560, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14244, cpu_share=0.0, mem_res=12232, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54800, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6313112, free=1034520, buffers=275440, cached=8552560, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.19, five_min_avg=1.33, fifteen_min_avg=1.35, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cbf80039cd32863ef153563fa00678eae15ec03e3b31e88af0d7521ec8fd9a1b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 57681} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '4b2aeae38e9530f071800ebf3551cfa9b44d1c120fc9ba0a1d1b725687b7782b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 57703} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8825156, partition_space_available_1k=72265328, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1646312, cpu_share=0.06, mem_res=1430812, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14292, cpu_share=0.0, mem_res=12280, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54796, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6315932, free=1029108, buffers=275492, cached=8555100, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.02, five_min_avg=1.26, fifteen_min_avg=1.32, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0ed0e62dc7c69cd4d5983417c391ef4ae680a0fd91bc23885517d203a1140173', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 58271} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '685d9abeb1ea3e72688251c3b3e446955c4f4bf2d22d1d7662bc277e150bcf61', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 58295} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8828652, partition_space_available_1k=72261832, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1653624, cpu_share=0.1, mem_res=1430668, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14292, cpu_share=0.0, mem_res=12280, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54804, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6321112, free=1015512, buffers=275564, cached=8563444, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.4, five_min_avg=1.82, fifteen_min_avg=1.5, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '3a4e3630dc9cdca92e21611c7db28753f9e26d9658333748dc73f47a1e27e3c9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 58830} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1aaa62dc53d6e92785ba46571780187005499d6ea715426025f6cba504e5408f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 58852} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8835244, partition_space_available_1k=72255240, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1658340, cpu_share=0.06, mem_res=1430932, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14256, cpu_share=0.0, mem_res=12244, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54812, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6320660, free=1009208, buffers=275632, cached=8570132, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.77, five_min_avg=1.63, fifteen_min_avg=1.45, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '3da478bc866b47d7c7c625f7cd302e55ed37eae7ae51d63ff6b769473312387a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 59395} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '360f0fac903030bcc00046ccf48050c0f3e20458d51513da637b1c97673da727', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 59416} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8841876, partition_space_available_1k=72248608, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1662344, cpu_share=0.06, mem_res=1430868, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14256, cpu_share=0.0, mem_res=12244, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54816, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6323800, free=998348, buffers=275688, cached=8577796, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.79, five_min_avg=1.66, fifteen_min_avg=1.48, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'eab8e6a68e5a0c1ccfb8ecbe389d299edd1b702f46f45feedbff10e171f56e97', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 59960} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8c9cdc8a6e179c34ebab39edc7b22f476f72278dfe6210ab24c2c15bff366cee', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 59981} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8846024, partition_space_available_1k=72244460, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1667248, cpu_share=0.07, mem_res=1430872, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14292, cpu_share=0.0, mem_res=12280, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54824, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6334908, free=982376, buffers=275752, cached=8582596, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.58, five_min_avg=1.62, fifteen_min_avg=1.47, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2898c2bbbb8ec3d63e7539d80cda6464890553fd217dad6d759aab6b84f4cdac', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 60528} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9a1c3bd3ad11df71645399c0b4692667e1735c91fc652466ba0d99445db203b2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 60549} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8854356, partition_space_available_1k=72236128, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1672116, cpu_share=0.07, mem_res=1430988, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14312, cpu_share=0.0, mem_res=12300, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54828, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6331276, free=957168, buffers=275836, cached=8611352, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.84, five_min_avg=1.71, fifteen_min_avg=1.51, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '25959203a97f9fa7f9a33b9504c9167d24b972477fff4eb2c776191d80c85f85', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 61152} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7cabd97267b705e3e4603053efb871a3939a12615a5bb9133f4f6b342b3d84cc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 61173} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8888380, partition_space_available_1k=72202104, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1698072, cpu_share=0.16, mem_res=1420616, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14312, cpu_share=0.0, mem_res=12300, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54832, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6337588, free=937088, buffers=275904, cached=8625052, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.35, five_min_avg=1.58, fifteen_min_avg=1.48, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c5d291da270f2e324b69c6188d4a9cecb40b5f4b85ed819a9896b4fefa245848', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 61767} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e5647b895acc141813aa9f3dd378abe44ae44a2d24c4827e3205d4f280b5f427', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 61790} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8892964, partition_space_available_1k=72197520, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1702048, cpu_share=0.09, mem_res=1419804, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14320, cpu_share=0.0, mem_res=12308, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54840, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6343284, free=926292, buffers=275956, cached=8630100, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.09, five_min_avg=1.7, fifteen_min_avg=1.52, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f77730ef771ace9378e82fb843e444fb16495476b9ecc8cccd3f45ef36c1f0df', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 62360} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ee538c61d559f940738274b9ec28882217e18c85901221f30700ec6cc562cdf9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 62381} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8901612, partition_space_available_1k=72188872, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1707448, cpu_share=0.07, mem_res=1419820, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14296, cpu_share=0.0, mem_res=12284, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54836, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6338180, free=922264, buffers=276016, cached=8639172, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.75, five_min_avg=1.69, fifteen_min_avg=1.53, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4c88082394811bf630c9381558588f55e0d24efa1afcb36df872b3b02c106744', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 62951} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '35b22a620e00af7f8357de7deaba530ccfdbda4afd39dbfe31ee12930aba2340', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 62973} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8908428, partition_space_available_1k=72182056, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1710952, cpu_share=0.06, mem_res=1419752, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14304, cpu_share=0.0, mem_res=12292, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54848, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6343412, free=910640, buffers=276056, cached=8645524, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.52, five_min_avg=1.62, fifteen_min_avg=1.52, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '12b6e0024870721a7b9ed8aa8b3e199069c9d737b92a56fabbcfd698a2f3cb33', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 63539} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2de6e376f1f9672b0b3af8559d82056e01fd9a1e3aa9050a7ef05c3dc6a5bcc4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 63558} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8918340, partition_space_available_1k=72172144, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1719324, cpu_share=0.08, mem_res=1421816, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14308, cpu_share=0.0, mem_res=12296, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54848, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6344104, free=899608, buffers=276088, cached=8655832, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.37, five_min_avg=1.56, fifteen_min_avg=1.5, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '45ebf3e83bea0a25c0b38a932c63c9b7e3e9175f420368d863cfe9d3d2a65747', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 64101} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ba018b3bfc4ef0b8db4e234b5924468d9e2907285899dd77e69f4892eef321ea', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 64123} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8919772, partition_space_available_1k=72170712, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1724256, cpu_share=0.06, mem_res=1422684, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14312, cpu_share=0.0, mem_res=12300, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54860, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6344896, free=897528, buffers=276120, cached=8657088, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.06, five_min_avg=1.42, fifteen_min_avg=1.45, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '30ede75cc1d475c397573477df247c177dbaa751ecbf785cc5de6d2e87a9a519', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 64657} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '65b8cdec76eed6803f6d055a9f7bf28587ac0145802e3bb60a7f60015bdb5849', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 64678} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8920492, partition_space_available_1k=72169992, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1728928, cpu_share=0.05, mem_res=1423728, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14316, cpu_share=0.0, mem_res=12304, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54860, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6350064, free=891440, buffers=276180, cached=8657948, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.32, five_min_avg=1.41, fifteen_min_avg=1.45, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '174adfbbeb4e1cd1ac7f4af0841a6f20af207a7cdea69bd777fe896f08a29113', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 65229} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '945cffe3ba5066958716fd4ac9ee1bc0ebe32b073227294db5cc71072133df73', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 65251} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8927244, partition_space_available_1k=72163240, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1733612, cpu_share=0.06, mem_res=1424676, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14332, cpu_share=0.0, mem_res=12316, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54868, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6358040, free=875040, buffers=276244, cached=8666308, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.66, five_min_avg=1.45, fifteen_min_avg=1.46, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '11049006d5b15e14498e38357dc95785e7cf2cd3274980a0fc9ac9186b9039ad', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 65810} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6cfe9ec613302c9f1e93bbf1fa2873be833cf855917536c813374006958f1dad', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 65831} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8935336, partition_space_available_1k=72155148, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1740656, cpu_share=0.07, mem_res=1426984, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14316, cpu_share=0.0, mem_res=12300, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54868, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6354532, free=871456, buffers=276324, cached=8673320, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.49, five_min_avg=1.43, fifteen_min_avg=1.45, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b34b2b6cdcac13319c4ee437186aee3b394b6ba96e24739dfa2b1ce4e6baa37a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 66368} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '66f6f6db1a7d9821ad6f9c9e205467a1fa0d9e4f9d1b2495a2829e3376045f72', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 66389} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8937196, partition_space_available_1k=72153288, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1746460, cpu_share=0.06, mem_res=1428304, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14320, cpu_share=0.0, mem_res=12304, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54876, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6356908, free=867172, buffers=276372, cached=8675180, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.03, five_min_avg=1.29, fifteen_min_avg=1.4, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '77437276d56bf33003af8b15be19e66a9e030b832b758fe99adef48cba21735b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 66933} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '052a0e80ba3864b830bf22d75e4b67688aa0ee713038b77cbd8ac815c22a6024', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 66953} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8944076, partition_space_available_1k=72146408, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1753304, cpu_share=0.06, mem_res=1430772, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14348, cpu_share=0.0, mem_res=12332, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54880, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6364020, free=852896, buffers=276436, cached=8682280, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.43, five_min_avg=1.36, fifteen_min_avg=1.41, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '953088908f8ea799648b4f4ead3122b1ed10eed71b24e952a61003576ca10712', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 67519} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cca5a923023b3c168a3eda3411ab6e2f4bf29439bdb76538b710421226f37358', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 67541} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8952760, partition_space_available_1k=72137724, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1761756, cpu_share=0.08, mem_res=1434212, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14384, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54884, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6374784, free=833352, buffers=276468, cached=8691028, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.1, five_min_avg=1.57, fifteen_min_avg=1.48, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '31f6d099d37f9ff9681f0bd0b881a4c9e45cb0b83ba1a7771f577b1b6f1dcb20', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 68086} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '56accb74b9d6d71f0d2c078b9ff91649efdfdb563d94dce66eba24e562b18422', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 68107} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8958388, partition_space_available_1k=72132096, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1776632, cpu_share=0.07, mem_res=1435408, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12372, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54892, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6380368, free=821300, buffers=276520, cached=8697444, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.51, five_min_avg=1.48, fifteen_min_avg=1.45, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2ae4e348beae7adbc95da55263e4b03eca5127a44566d722b71b3c0c0b83eae9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 68649} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5351ef075ef595a27f5830ffaeca755f750778727d60bfc10c0cb62a1e26dcb9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 68672} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8965336, partition_space_available_1k=72125148, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1780756, cpu_share=0.06, mem_res=1435296, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12372, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54896, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6381308, free=813304, buffers=276564, cached=8704456, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.87, five_min_avg=1.78, fifteen_min_avg=1.55, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '45719e6705f0111697213455a2c9a7f6d9373bb1e8424c7285bd91f437885295', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 69219} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '14efd6262989030196bd66e4603bab6e3e719beeb0820ff5f20ebfebbf664965', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 69243} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8971876, partition_space_available_1k=72118608, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1784780, cpu_share=0.06, mem_res=1435224, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14368, cpu_share=0.0, mem_res=12352, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54900, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6382576, free=805048, buffers=276624, cached=8711384, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.53, five_min_avg=2.08, fifteen_min_avg=1.66, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f4441ac7f0c564d5df961f2af64756a276bf6f9914cc53033765bd8575fee48a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 69872} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a4b3403eb303403b2090f73db95e04692c1f3e3ec8c7437f317a49af4d0bf2fc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 69894} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8986716, partition_space_available_1k=72103768, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1792492, cpu_share=0.13, mem_res=1435428, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14392, cpu_share=0.0, mem_res=12376, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54904, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6388244, free=784016, buffers=276716, cached=8726656, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.29, five_min_avg=1.99, fifteen_min_avg=1.65, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'da1ca1b82d0f61f51d2a2f280e37115ea2c3fe07d77d63b2e36dcfd9f208b5e1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 70530} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f88a5f34fa45fdb5c4f8534ff93eedc54332a499d8d273b154062e5286907e3c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 70551} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8994176, partition_space_available_1k=72096308, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1801720, cpu_share=0.13, mem_res=1435308, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14404, cpu_share=0.0, mem_res=12388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54908, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6391648, free=772448, buffers=276784, cached=8734752, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.59, five_min_avg=2.11, fifteen_min_avg=1.72, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '69622996d3f5595e6016d21fdfc312740ef5f9f1805a959462e058eb279b0a1e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 71094} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5af91e58642d893d1d865a61198e3201f79e8362c28bbe4125f8e5f89f922317', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 71116} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9002468, partition_space_available_1k=72088016, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1806668, cpu_share=0.06, mem_res=1435372, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14404, cpu_share=0.0, mem_res=12388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54916, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6387728, free=768068, buffers=276844, cached=8742992, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.73, five_min_avg=1.94, fifteen_min_avg=1.68, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5d7509aa7f34a7e4d3a33cdc66670eead3e53806268d78785fc74ff729619441', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 71647} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a6d84de838bd4d62b5b41bfcfc2677d908f53e4eb78d544b3a70e9410df73473', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 71668} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9010020, partition_space_available_1k=72080464, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1811156, cpu_share=0.06, mem_res=1435308, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14404, cpu_share=0.0, mem_res=12388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54920, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6395476, free=752864, buffers=276892, cached=8750400, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.49, five_min_avg=1.82, fifteen_min_avg=1.65, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '86de8a93e822c5c522d309b4b1ce5df42e69e2184a260a6018b1e73c32b9bdde', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 72269} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6818b00824fcb2a5416bf99ee4b445e407647f7b38c995f434b3c157d3ee0487', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 72292} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9043516, partition_space_available_1k=72046968, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1838736, cpu_share=0.14, mem_res=1427180, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14408, cpu_share=0.0, mem_res=12392, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54928, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6384804, free=729092, buffers=277068, cached=8784668, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.71, five_min_avg=2.11, fifteen_min_avg=1.76, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0e1edd3e8b976b709b028a1438b1dbf3df99e5456a0f73919328549f30e1fc0f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 72836} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e4e9774fff16f9c284f6553c196b8a385a0f322fc8bb8efc23dcfa6a508cef43', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 72859} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9049836, partition_space_available_1k=72040648, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1841324, cpu_share=0.06, mem_res=1426064, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14372, cpu_share=0.0, mem_res=12356, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54932, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6386580, free=721068, buffers=277096, cached=8790888, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.89, five_min_avg=1.97, fifteen_min_avg=1.73, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7049fea73af644f87b01731a97b412fe8e384f55363905c3760375c644dea15e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 73399} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '59cb75d22f794462eb35ef6e619c21908990a8ab5e295080d073c735d8bb1c96', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 73420} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9050248, partition_space_available_1k=72040236, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1844344, cpu_share=0.06, mem_res=1425932, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14372, cpu_share=0.0, mem_res=12356, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54940, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6389212, free=717740, buffers=277152, cached=8791528, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.26, five_min_avg=2.04, fifteen_min_avg=1.77, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '761e826be8830b91475233a6a299d915e09d0c4b7b0f0b6fec1ddebbeddb5dbd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 73994} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8a6613257f8d0a551818c1ea2db0b98a196ab5907b9fc46588c270a1e41e1360', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 74014} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9060032, partition_space_available_1k=72030452, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1850648, cpu_share=0.08, mem_res=1425864, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14384, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54940, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6392648, free=703324, buffers=277200, cached=8802460, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.62, five_min_avg=1.89, fifteen_min_avg=1.73, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c9694073e6da9950dd7093e5d1b59c1e7ece5c7b8185abd9a83d542d9aed22ac', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 74560} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2bea02c7d0be915aac59e3d66693728ef7b24b7fe98564c089fa252d9608b254', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 74581} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9061512, partition_space_available_1k=72028972, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1854256, cpu_share=0.05, mem_res=1425936, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14412, cpu_share=0.0, mem_res=12396, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54948, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6390880, free=704668, buffers=277236, cached=8802848, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.23, five_min_avg=1.73, fifteen_min_avg=1.69, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f5d3ec955a742046524475966a14c445f766053a0c0172a7df04d11a147e67a3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 75117} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f66d358874f5ab2b616aa3734a36d4769692c675a840f02ef2bde54b85e3a01b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 75138} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9067672, partition_space_available_1k=72022812, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1858348, cpu_share=0.06, mem_res=1425936, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14412, cpu_share=0.0, mem_res=12396, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54952, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6396492, free=692536, buffers=277284, cached=8809320, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.15, five_min_avg=1.84, fifteen_min_avg=1.72, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '882d5096dced004c160275e45424087588e7b820a35820a0ac8fd288f6a0b6cf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 75710} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b835307a64760ed4170f048d8001b0fbe16bd24f767f5840d4071afa8285c046', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 75732} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9076272, partition_space_available_1k=72014212, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1863580, cpu_share=0.08, mem_res=1425728, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14424, cpu_share=0.0, mem_res=12408, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54956, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6398944, free=680544, buffers=277344, cached=8818800, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.66, five_min_avg=1.75, fifteen_min_avg=1.7, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dd7d4b20a59c5cd1cdc1465c5e9c20d1fbf9fbe5895a16e2d2c586cee11c4eee', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 76325} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '862fa30f21de417423706de84dda7c8a1bb09b3e88671be118b4b162db3c3ae4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 76345} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9080512, partition_space_available_1k=72009972, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1869464, cpu_share=0.09, mem_res=1426064, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14380, cpu_share=0.0, mem_res=12364, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54960, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6402152, free=672612, buffers=277428, cached=8823440, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.25, five_min_avg=2.09, fifteen_min_avg=1.81, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '66e337fae85ed67ebf740cb39468b5c87c27a83cb2c33b92dedd27abba131444', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 76909} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e346b92ab31ee25d41d8492be484128ff849199e313ec06e25febd2a67680f49', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 76930} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9084912, partition_space_available_1k=72005572, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1875968, cpu_share=0.08, mem_res=1426976, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54968, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6401008, free=669300, buffers=277508, cached=8827816, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.91, five_min_avg=1.92, fifteen_min_avg=1.77, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5d523c3b2aad3213c49b4c9bc8ca8ac5ec319fbbfd6863ed2f133a26b0c9266d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 77468} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '750d32294f0f20bc37c6155821c6a4f4af5d5af70eb967365b380b7724ce7777', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 77490} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9090948, partition_space_available_1k=71999536, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1879772, cpu_share=0.06, mem_res=1427064, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14404, cpu_share=0.0, mem_res=12384, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54972, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6406196, free=658488, buffers=277548, cached=8833400, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.28, five_min_avg=1.73, fifteen_min_avg=1.71, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '713e3ab6b1b01db91859b58177f7af2c95ccbc38d1749cbfa7cf2263852a21f5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 78026} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e6d1d1cf5bf659af3ca6a6454580499abc3beb95e9c57ef966c6542de8352297', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 78048} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9097212, partition_space_available_1k=71993272, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1885568, cpu_share=0.06, mem_res=1428800, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14404, cpu_share=0.0, mem_res=12384, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54976, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6400940, free=657424, buffers=277580, cached=8839688, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.15, five_min_avg=1.61, fifteen_min_avg=1.67, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5d7061c58c63ebeade52f497f859612c380e672cd6ef586c15ff550c92b8b48b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 78595} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '4c08645dfe29b979938915d9ad2361f7d68cd3ce180a0e8e78de6a479917c55b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 78616} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9098808, partition_space_available_1k=71991676, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1891728, cpu_share=0.06, mem_res=1430308, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14408, cpu_share=0.0, mem_res=12388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54980, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6404564, free=651796, buffers=277640, cached=8841632, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.03, five_min_avg=1.49, fifteen_min_avg=1.62, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2dd3989abc6f3d8c1ce0b8a3de014fa1663b9f557da96ed0c1a52840b03eaa9b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 79287} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c55815553f9f817139945a096231b1552f60874f0cf0e8e7bde2ad1bc624f7e6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 79308} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9110660, partition_space_available_1k=71979824, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1901896, cpu_share=0.09, mem_res=1433352, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14408, cpu_share=0.0, mem_res=12388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54984, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6415376, free=628416, buffers=277700, cached=8854140, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.41, five_min_avg=1.5, fifteen_min_avg=1.62, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4cc0468cf53d32d82c37a710097876f6fd92c1f99581a455dc6962a3a54c4b05', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 79899} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '04c00523d6201a5a5f5ebeb173c218aad4d6f4421c190eec054e2507c45aba7d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 79922} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9114660, partition_space_available_1k=71975824, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1909668, cpu_share=0.07, mem_res=1436088, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54988, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6419316, free=620196, buffers=277728, cached=8858392, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.28, five_min_avg=1.47, fifteen_min_avg=1.6, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5bdfb92b25a96a83abecdcf309dbe97508fd591123f7eb804a13deef32ce1363', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 80468} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '14c1ff5ae60e6cc3ad8f8188ffca123682b1e1938be2c0947e31fbe507c781ad', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 80490} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9158480, partition_space_available_1k=71932004, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2079836, cpu_share=0.19, mem_res=1548172, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54988, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6524720, free=441304, buffers=277844, cached=8931764, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.23, five_min_avg=1.4, fifteen_min_avg=1.57, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '128c4b50e843f4d60e373d9f8b729e4eaeba4d6a788786f656e8e9c10e0ee32a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 81035} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '44b352ff37361938cc22b099d339221ba94f1507177be8e8369328ba3756edd0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 81057} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9127156, partition_space_available_1k=71963328, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2034396, cpu_share=0.21, mem_res=1547664, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54996, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6526148, free=500432, buffers=277888, cached=8871164, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.24, five_min_avg=1.36, fifteen_min_avg=1.54, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4e07a1de1cbcb670e4e79933b88c5db6c03ddeddea3f6c12870c8f997afc27c1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 81592} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'aae4e6a895d2bfdf9882b33ae5601fc2ea43cf3cbe43026e82669c8d0961105a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 81612} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9133396, partition_space_available_1k=71957088, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2038352, cpu_share=0.06, mem_res=1547484, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55004, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6529200, free=491020, buffers=277916, cached=8877496, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.62, five_min_avg=1.41, fifteen_min_avg=1.55, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8d45e63294a13156a011772ed1148dd8fb278ac2b207712c5224fdccfb9bf244', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 82180} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '238152f7e60b9d81d3b448c925b359e01ca6d6829efb4a1e0b21bf561c209b2b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 82200} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9141996, partition_space_available_1k=71948488, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2043244, cpu_share=0.08, mem_res=1547496, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14396, cpu_share=0.0, mem_res=12376, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55008, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6527168, free=484496, buffers=277948, cached=8886020, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.16, five_min_avg=1.32, fifteen_min_avg=1.51, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c961070ec9a705a4f163941571971d3a8dfd81edba197af6b9d68c01ebd80747', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 82768} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '303a88bb4bb9ecc07f2c39f627e9a641c9454b028af351d5ff504dd561560eee', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 82793} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9147196, partition_space_available_1k=71943288, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2049580, cpu_share=0.09, mem_res=1547712, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14408, cpu_share=0.0, mem_res=12388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55012, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6535964, free=469652, buffers=278000, cached=8892016, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.14, five_min_avg=1.28, fifteen_min_avg=1.48, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e4f0103bc059959474a9cd9a429f2e042b785c3f08661448d6e2f63e5010eb65', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 83332} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5540c0923a2f1863552761bde4c2c2d7b2326e1d3f4dd7f5ceb020e972563c55', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 83353} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9153828, partition_space_available_1k=71936656, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2053816, cpu_share=0.06, mem_res=1548128, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14388, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55016, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6536748, free=462624, buffers=278016, cached=8898244, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.1, five_min_avg=1.23, fifteen_min_avg=1.45, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'abac4e1c58e9ff2cf2009b1ced38127308e75b32bc4dbdbc93806f29ac1962d7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 83884} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '94af3e87c37488491a5bd9db381b0668040df890990e943005db961e61453bb5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 83911} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9155832, partition_space_available_1k=71934652, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2060596, cpu_share=0.06, mem_res=1549980, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14400, cpu_share=0.0, mem_res=12380, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55016, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6544904, free=450616, buffers=278040, cached=8902072, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.3, five_min_avg=1.26, fifteen_min_avg=1.45, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4c6f8819eb427a9352bcbc925f73e1cd094895300ad734417f788bcb508cf153', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 84494} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f6569828b1ed2cffec3bf9301b845142d0967e9e6bc7e283664c28d382aaae92', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 84517} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9176804, partition_space_available_1k=71913680, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2082640, cpu_share=0.14, mem_res=1553640, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14420, cpu_share=0.0, mem_res=12400, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55028, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6542264, free=433232, buffers=278068, cached=8922068, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.08, five_min_avg=1.2, fifteen_min_avg=1.41, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a9f5173638fd6d5065bfa3b291d5e4d6873109e8752a586e4a0b7fec0e23df7c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 85074} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8daebe6cb983e9245d0f90505bc215a68455c46a5f9b4f688090b73c64a497b4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 85095} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9178164, partition_space_available_1k=71912320, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2085708, cpu_share=0.06, mem_res=1552892, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14420, cpu_share=0.0, mem_res=12400, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55036, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6546024, free=428160, buffers=278092, cached=8923356, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.02, five_min_avg=1.17, fifteen_min_avg=1.39, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '48d55eb5dc5628578cede347c622217a619a57b5c539a987aa082fd07ed07bae', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 85666} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ff06743b1d72676eb1661a09c574ae240deca6c3d758814c180b9c38a5734104', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 85687} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9185140, partition_space_available_1k=71905344, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2089916, cpu_share=0.07, mem_res=1552956, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14420, cpu_share=0.0, mem_res=12400, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55036, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6549048, free=417052, buffers=278132, cached=8931400, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.05, five_min_avg=1.14, fifteen_min_avg=1.36, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '16f2a33b5d0d63736a9c10f12fcfa1df9754112615a767700c7cab7589c7b79c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 86219} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '684c38ec342ef4472e482f02f0940467e665484da61ef6b10cfd625c8878519f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 86241} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9192100, partition_space_available_1k=71898384, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2093600, cpu_share=0.06, mem_res=1552856, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14424, cpu_share=0.0, mem_res=12404, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55040, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6547724, free=412340, buffers=278176, cached=8937392, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.99, five_min_avg=1.1, fifteen_min_avg=1.33, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '79059c252517aba415dc9367c74a0402c0c217add414c34c20df37cbb87c816b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 86940} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a111151741f8eb02653b6cf70f7d878fc59c490fbd719e8b5d18959cbe7da211', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 86963} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9195320, partition_space_available_1k=71895164, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2099308, cpu_share=0.08, mem_res=1552872, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14404, cpu_share=0.0, mem_res=12384, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55048, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6549488, free=406704, buffers=278240, cached=8941200, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.96, five_min_avg=1.08, fifteen_min_avg=1.31, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '44f5bfc52f015b0c70984590ff1633b55ba7f5bfd20a60983859f5f596716387', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 87608} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a626970d7fbd37fea100441478b6c1537554427e6286175580b20a79677607ab', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 87628} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9197664, partition_space_available_1k=71892820, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2123712, cpu_share=0.32, mem_res=1571116, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=45560, cpu_share=0.01, mem_res=43340, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55048, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6600352, free=353648, buffers=278272, cached=8943360, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.63, five_min_avg=1.22, fifteen_min_avg=1.34, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '00a53805a90686868a0339876b6139d0e01ef063e0464c01096217878bfb68ac', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 88212} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1610c01f671f4b2af296516df83af1dd26b0d845396ce2356b9a27ab243fcfbc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 88231} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9206708, partition_space_available_1k=71883776, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2129940, cpu_share=0.09, mem_res=1571680, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=46412, cpu_share=0.0, mem_res=44196, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55056, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6608056, free=335604, buffers=278312, cached=8953660, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.91, five_min_avg=1.4, fifteen_min_avg=1.4, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '536e19114ab4aadf9c53f87f9d7aed70089bbff28d7c3da87e1ef0405f45b76f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 88780} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'fb5c8b597ae02e6213adce62d9d891e390328ab1a75343fd4fe5f86ee286d152', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 88801} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9215320, partition_space_available_1k=71875164, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2134700, cpu_share=0.08, mem_res=1571332, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=46532, cpu_share=0.0, mem_res=44288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55064, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6603388, free=331944, buffers=278388, cached=8961912, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.2, five_min_avg=1.28, fifteen_min_avg=1.36, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '49539bcfa06cd8863378045f521d538a2bcc1235c5f3fc0ba4f90a9a9d12ea0c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 89343} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2817aafe6e7e5448f3b39a789e961d8aa9493253ef9bf3dbdc4aa88e7235440e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 89368} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9217104, partition_space_available_1k=71873380, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2139044, cpu_share=0.06, mem_res=1571548, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=46540, cpu_share=0.0, mem_res=44304, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55064, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6604440, free=329556, buffers=278444, cached=8963192, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.03, five_min_avg=1.21, fifteen_min_avg=1.33, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6eeec23ca65318f9f2ba781d38586eb84fccac9af9284b31627cfede42e7dd8f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 89917} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7557cd4f7e94330d7045e2067395dc6bc1c204bbc57c3c56d81cfc2e48aa85b3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 89938} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9224200, partition_space_available_1k=71866284, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2145576, cpu_share=0.07, mem_res=1572648, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=46540, cpu_share=0.0, mem_res=44304, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55068, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6608060, free=318368, buffers=278488, cached=8970716, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.71, five_min_avg=1.36, fifteen_min_avg=1.37, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '42cf7d538758393bdfa96d706607572385e358c6245d83665fde48a7dc33e69f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 90474} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ae08b098e70e6f3af62d38745040065718589ec8f9f3004cf98af18e5283918a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 90495} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9229840, partition_space_available_1k=71860644, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2149492, cpu_share=0.06, mem_res=1573272, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42088, cpu_share=0.0, mem_res=39852, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55076, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6603144, free=317676, buffers=278528, cached=8976284, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.92, five_min_avg=1.44, fifteen_min_avg=1.39, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '61fc9157b1b88f4c60211aec1e061bc3048e2059ce92fc2e97df1105cab235c8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 91098} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ba730df30fedd8c2be4ca0140dd500b34de8e262c6b310733d64256f207dc783', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 91119} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9257716, partition_space_available_1k=71832768, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2188500, cpu_share=0.16, mem_res=1578060, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42620, cpu_share=0.0, mem_res=40360, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55080, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6616612, free=275344, buffers=278632, cached=9005044, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.64, five_min_avg=1.46, fifteen_min_avg=1.4, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0808aeabadc04c379f86651971560c97f533c4d51d5e6e409a4bcc66dc196774', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 91665} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '02a42fc1540f48d69c0711b4efae34666a047159bb7aa7a6f2bc2b04fdd8146a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 91686} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9264000, partition_space_available_1k=71826484, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2191724, cpu_share=0.07, mem_res=1577460, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42620, cpu_share=0.0, mem_res=40360, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55088, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6615700, free=269184, buffers=278656, cached=9012092, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.28, five_min_avg=1.39, fifteen_min_avg=1.38, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'add60410517f40dd3e42d7d190200539174bc8189b2805ea5e7dff931de46193', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 92220} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f58f27972978e21a5cdc0537ed3a7b9f5aed9c081958b8860b383f4fa9ee7e66', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 92241} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9270384, partition_space_available_1k=71820100, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2195472, cpu_share=0.06, mem_res=1577472, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42672, cpu_share=0.0, mem_res=40412, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55088, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6612280, free=266648, buffers=278700, cached=9018004, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.25, five_min_avg=1.36, fifteen_min_avg=1.37, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7437381467f3efbefc17675a51acb03e06b13b6ecb95f8b59cf4166fc242457d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 92814} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e7fe2f89eeba9ed7278b92b9b4e27662de7212aff3b293ab30443f02dbbf73d5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 92835} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9278768, partition_space_available_1k=71811716, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2200488, cpu_share=0.08, mem_res=1577672, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42732, cpu_share=0.0, mem_res=40472, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55092, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6618264, free=251872, buffers=278748, cached=9026748, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.1, five_min_avg=1.3, fifteen_min_avg=1.34, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '08b63ef866f2c2a23ad24becbed86e94ba07ba99b0e8cc98e9be5d8e751e8ea5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 93375} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '06d6f08e1e94eb6943cf4dbb21e26e83de7e78ab360e00364e983b0f32356a13', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 93396} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9279952, partition_space_available_1k=71810532, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2204320, cpu_share=0.06, mem_res=1577616, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42760, cpu_share=0.0, mem_res=40500, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55092, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6613452, free=255476, buffers=278788, cached=9027916, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.15, five_min_avg=1.47, fifteen_min_avg=1.4, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a5be12c3141a7377cd2a608fca4ce66527b3c80df2e9d54d53e060770602c36f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 93936} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a5e34f43f16d38e033c1f487821e1cc7c45d529cb2a9589d003843e21cb6c504', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 93959} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9285424, partition_space_available_1k=71805060, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2207896, cpu_share=0.06, mem_res=1577680, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42188, cpu_share=0.0, mem_res=39928, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55112, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6616628, free=246504, buffers=278820, cached=9033680, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.41, five_min_avg=1.38, fifteen_min_avg=1.37, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '13a0b9c2069a598b3f724a62fa32497a63f8b35249ff7fab7d6fe2f95a08f373', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 94499} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0fb96d1138b9ed21530a3c004985cea3d8940be5a39137e26eb4ec47dc0bd279', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 94520} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9288336, partition_space_available_1k=71802148, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2213444, cpu_share=0.07, mem_res=1577736, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42236, cpu_share=0.0, mem_res=39976, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55112, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6621232, free=238320, buffers=278852, cached=9037228, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.17, five_min_avg=1.31, fifteen_min_avg=1.34, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e21e1fd85eaca8a944b02534762258dedcb91a51454b0c304e824b9614d7c037', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 95084} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '96cf5c6f3d77d08524ca27e44bafbf805dc3bec89eedf9ce5915bdbd96aecfd0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 95107} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9298292, partition_space_available_1k=71792192, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2219452, cpu_share=0.08, mem_res=1577836, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42308, cpu_share=0.0, mem_res=40048, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55116, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6621944, free=227888, buffers=278904, cached=9046896, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.03, five_min_avg=1.24, fifteen_min_avg=1.32, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '54c791e295685899faf7d9d75381f032f54aca21cfda737508a2924f407fe19a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 95647} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3457bc5ab6e6910e2fc922f242789fc67bde60a5e29e03542bd01403f059eaa7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 95669} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9299420, partition_space_available_1k=71791064, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2224384, cpu_share=0.06, mem_res=1578908, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42380, cpu_share=0.0, mem_res=40120, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55124, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6619324, free=229480, buffers=278912, cached=9047916, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.02, five_min_avg=1.21, fifteen_min_avg=1.3, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c8f8592d7b532608d6215bd7a18237ecf24a88281b7fa73fe4f8a415ca33b673', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 96213} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '272352ce5740a25b82e21eae82f3ffcd38afa23348e8e4f5a56cf08c10c138a4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 96237} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9306388, partition_space_available_1k=71784096, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2230140, cpu_share=0.06, mem_res=1580200, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42472, cpu_share=0.0, mem_res=40212, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55132, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6609892, free=237760, buffers=278944, cached=9049036, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.18, five_min_avg=1.23, fifteen_min_avg=1.3, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f66e27d8ffdbb72bcd53c83e1f74b04bc35b84b6ad242d78f680f5b8ad7d1a46', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 96862} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2234af3e2e03f36cd571696079532a4fd72c2728916efba744190c2599a82722', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 96883} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9317608, partition_space_available_1k=71772876, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2236800, cpu_share=0.09, mem_res=1580660, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42472, cpu_share=0.0, mem_res=40212, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55132, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6601264, free=239024, buffers=278964, cached=9056380, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.46, five_min_avg=1.34, fifteen_min_avg=1.34, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0c85bc9c6b1285db4ada6f8aa581a41662a3b6270391a432d5c39e492da3f9a6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 97418} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8637adae35311cce0f95e983d65d44cb7879a6f2d52936c60bfed1dfbb445beb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 97440} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9314008, partition_space_available_1k=71776476, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2241436, cpu_share=0.07, mem_res=1580768, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42196, cpu_share=0.0, mem_res=39936, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55136, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6609144, free=233296, buffers=279000, cached=9054192, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.72, five_min_avg=1.42, fifteen_min_avg=1.36, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '23ce50022ceb59fd29d3999e62cafa5dc5c35204dcd1c9598a5e819eb6ace990', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 97970} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '58a8867e32abe8ce9c9871011d98f33727b9b74ccada20455939c21b1ec18035', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 97993} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9321040, partition_space_available_1k=71769444, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2246752, cpu_share=0.05, mem_res=1581828, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42252, cpu_share=0.0, mem_res=39992, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55224, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6599432, free=242864, buffers=277576, cached=9055760, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.28, five_min_avg=1.35, fifteen_min_avg=1.34, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dba2000fe27d01ccbf2a0f37f7e1de829fdf89e4108a4a8fa8a101a939236d22', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 98535} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '79cb70e1fcb87256019e1354a1141cf9351de5e32cd2d85565cc0438c2f3f439', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 98558} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9349012, partition_space_available_1k=71741472, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2274152, cpu_share=0.1, mem_res=1583528, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42288, cpu_share=0.0, mem_res=40028, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55304, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6600348, free=232768, buffers=275724, cached=9066792, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.33, five_min_avg=1.38, fifteen_min_avg=1.36, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9daad354c361b4a25c3412f06397a1c51b7e3dc8a09c606a974db266a01fc36d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 99097} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '381403f22d490c35052136068dd566a24685a4f9de743247647d4557923ab82b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 99119} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9354728, partition_space_available_1k=71735756, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2276436, cpu_share=0.06, mem_res=1582744, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42404, cpu_share=0.0, mem_res=40144, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55308, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6608316, free=222684, buffers=275452, cached=9069180, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.22, five_min_avg=1.34, fifteen_min_avg=1.34, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a0ed13b20468e8b8e5c4c50599a821a02da6b120032c83fd8bb4b33d9df1fe0d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 99799} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2659eb7f128bc5812a152b02842df7bbcb7ca15ce976c32cab20764ce7194e35', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 99820} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9364052, partition_space_available_1k=71726432, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2284176, cpu_share=0.11, mem_res=1582840, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42452, cpu_share=0.0, mem_res=40192, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55308, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6628536, free=214128, buffers=274268, cached=9058700, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.05, five_min_avg=1.57, fifteen_min_avg=1.42, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5af2810dc42d58ea903339623ab9a42d679c1f638a4daa67e05d0f64998fdbbb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 100399} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f8a829cd0985cce7ed234862414a23eebcffa442c13ff099bb457530bd14d37d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 100420} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9378244, partition_space_available_1k=71712240, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2293392, cpu_share=0.14, mem_res=1582976, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42472, cpu_share=0.0, mem_res=40212, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55324, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6615180, free=223208, buffers=273820, cached=9063424, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.82, five_min_avg=1.9, fifteen_min_avg=1.55, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b4391604d5ed8e8392f5182d39b9f6154b324ab0a7455563bdd10941f02226a0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 101011} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'affeb8f73c62414a07a99455825a3c27d0a1fc25adc376464ffc578e9ca60153', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 101035} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9387284, partition_space_available_1k=71703200, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2298376, cpu_share=0.08, mem_res=1583932, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42188, cpu_share=0.0, mem_res=39924, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55320, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6622200, free=214776, buffers=273744, cached=9064912, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.04, five_min_avg=1.84, fifteen_min_avg=1.55, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4298e91e3b59ca3b9f94b71f92509bd5b6dfb2809fea9f8a75a6d4eeb78670a2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 101648} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0bc17742aa2afdfa67ff359e3b2e29538f37d1b4c50b2b9af6df04ad9f939c1a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 101668} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9402852, partition_space_available_1k=71687632, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2466868, cpu_share=0.15, mem_res=1751960, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42188, cpu_share=0.0, mem_res=39924, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55696, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6784976, free=213680, buffers=273640, cached=8903336, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.56, five_min_avg=1.76, fifteen_min_avg=1.54, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 03:58:29] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6756 0.002305 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8eb736832342ee1c76c0015d5649051c0e052fd4541d09886d25328efc2ffeff', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 102283} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '797db620daffeea3c27be0d1406aa4fcce6dde0d7a571d869d122d5f2d04bd3c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 102306} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9411720, partition_space_available_1k=71678764, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2468308, cpu_share=0.08, mem_res=1750536, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42272, cpu_share=0.0, mem_res=40008, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55696, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6776760, free=231340, buffers=273488, cached=8894044, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.13, five_min_avg=1.61, fifteen_min_avg=1.5, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7d1ff5751e90942546741052ee1f0e74e2b54fb989267a709d42e14f90a9d1be', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 102850} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9aadabb473b5ec4dd7edd24fedc810dd5120f508b66944a3673c4678164e6e32', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 102869} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9412756, partition_space_available_1k=71677728, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2471992, cpu_share=0.06, mem_res=1750544, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42276, cpu_share=0.0, mem_res=40012, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55696, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6767836, free=239460, buffers=273548, cached=8894788, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.12, five_min_avg=1.53, fifteen_min_avg=1.48, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1dbc3dfe6299a058092c870dc2dd311f7fa59c03089b4b1da32dc5932aef34f3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 103419} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '376ca7afb30a6b9f6f8483b46e751e9b72b8996106ef40ca542d14c2299a375d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 103439} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9418864, partition_space_available_1k=71671620, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2474624, cpu_share=0.07, mem_res=1750700, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42368, cpu_share=0.0, mem_res=40104, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55700, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6774424, free=241708, buffers=273200, cached=8886300, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.25, five_min_avg=1.5, fifteen_min_avg=1.47, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '90a979d1fc5b4c4fc02324fa44bc024316fb03f8ad5e23a462624fd727889f92', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 104005} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '427832c8567379b82597339bc3ed26e25d1915fc482d965c889dc65f225150ca', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 104028} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9421920, partition_space_available_1k=71668564, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2480740, cpu_share=0.07, mem_res=1752720, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42412, cpu_share=0.0, mem_res=40148, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55700, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6786324, free=233468, buffers=273112, cached=8882728, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.32, five_min_avg=1.48, fifteen_min_avg=1.46, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c68e99a69fac2019aefd22192f0a2595110e10158988c01a799604f86b26de8d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 104808} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '17edc791d102ba8c7d2f799e2ac9668fac3700bef25f2192936928ab6a50b3bf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 104832} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9428508, partition_space_available_1k=71661976, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2485192, cpu_share=0.06, mem_res=1753600, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42176, cpu_share=0.0, mem_res=39912, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55708, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6759060, free=252896, buffers=273472, cached=8890204, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.49, five_min_avg=1.7, fifteen_min_avg=1.54, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 04:04:14] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6756 0.001960 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '72a829226d139716f315e0935887a0340487ae8767a596a57ae8decadd004eba', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 105684} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '520e63bcd7d4bf4566f637991f78649e8fe3242c5d7772aaca0f30f74cf37f54', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 105706} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9436016, partition_space_available_1k=71654468, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2491596, cpu_share=0.07, mem_res=1755780, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42328, cpu_share=0.0, mem_res=40064, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55712, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6758088, free=245528, buffers=273556, cached=8898460, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.47, five_min_avg=1.56, fifteen_min_avg=1.5, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c285df83687b0a70330a73a645b8a45f88bd2a86d200fcd0aaa732feb770239a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 106245} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ee1450333b4d2c19d2cd2040b58d890a0db2f845b481d7a8a5465d8c45c08a28', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 106267} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9437828, partition_space_available_1k=71652656, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2496856, cpu_share=0.06, mem_res=1757064, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42348, cpu_share=0.0, mem_res=40084, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55720, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6770088, free=232248, buffers=273596, cached=8899700, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.11, five_min_avg=1.44, fifteen_min_avg=1.46, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c6670e15e39dcab8b8dec2e6f32d44b6db60c46ac3f85017ef8dc1e5ccf7bfaa', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 106808} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'df73ee4b5851cfcf229a7dfd9973ea9dc50eca6e848eb694b6e41f70ff103da1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 106830} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9438032, partition_space_available_1k=71652452, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2500348, cpu_share=0.06, mem_res=1757620, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42460, cpu_share=0.0, mem_res=40196, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55720, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6762972, free=238284, buffers=273628, cached=8900748, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.37, five_min_avg=1.45, fifteen_min_avg=1.46, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a400d8ad43600606f422da924485bc159b8109166bcbc4d33c0f32dcf11c9c6b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 107406} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'be987b4e4a17bbe2611dbda2d266388db2aa3d44dee17d8393a8f42dda858257', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 107426} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9445368, partition_space_available_1k=71645116, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2504016, cpu_share=0.07, mem_res=1758032, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42460, cpu_share=0.0, mem_res=40196, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55856, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6768320, free=233088, buffers=273376, cached=8900848, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.57, five_min_avg=1.49, fifteen_min_avg=1.47, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '884b04a7a8fc39081ee72eea0ac6ceaa7aad0d298dfc70ee5a6447ceaa28e166', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 107991} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '16eeedd0ba952849382e6241564b0a7a697765485cc253fe808db17936a10cd4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 108012} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9452932, partition_space_available_1k=71637552, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2509588, cpu_share=0.08, mem_res=1759848, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42504, cpu_share=0.0, mem_res=40240, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55856, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6769784, free=237880, buffers=273032, cached=8894936, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.27, five_min_avg=1.42, fifteen_min_avg=1.45, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '13f3b196d1a144782b8844d749d1eb9be59648683f8941f1ca346e684955ec03', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 108617} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c1b87c2ea825ab96c81a4544d30d1c2078d47cc7696297d77f8d80052de9a80e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 108638} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9457128, partition_space_available_1k=71633356, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2514240, cpu_share=0.08, mem_res=1759000, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42168, cpu_share=0.0, mem_res=39904, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55856, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6783756, free=218768, buffers=273048, cached=8900060, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.0, five_min_avg=1.8, fifteen_min_avg=1.57, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd2b0004cb72a27d54e815c6f47bea324cbd21d454b9e835823e39298f99e0634', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 109173} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a249e0024df7d48341acf3478dbc0e1a59a852aeee0fe0bde35e3783f1ebc61b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 109194} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9462896, partition_space_available_1k=71627588, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2518388, cpu_share=0.06, mem_res=1759940, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42248, cpu_share=0.0, mem_res=39984, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55864, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6764196, free=235256, buffers=272924, cached=8903256, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=1.65, fifteen_min_avg=1.53, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'de3dc4bc7f9cc3b9df268063f96b7993fa6de3a86af407d381c7d9e11ec87c02', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 109731} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9bc998df5223775e2b1021b95876bf1385c8d4a528c06b9dad61d9b21e6946f7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 109752} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9468152, partition_space_available_1k=71622332, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2523728, cpu_share=0.06, mem_res=1762444, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42296, cpu_share=0.0, mem_res=40032, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55868, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6768004, free=228080, buffers=272804, cached=8906744, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.75, five_min_avg=1.69, fifteen_min_avg=1.55, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'db3e5ccffbe82e15d5e1580970bd974d8a7dd91429b33ae6f3f1171409c035a6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 110284} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '030f3f65565482978258d8a414addf84acea0486676f1140243834c7b5f76028', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 110305} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9469496, partition_space_available_1k=71620988, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2527368, cpu_share=0.06, mem_res=1763704, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42344, cpu_share=0.0, mem_res=40080, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55872, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6768404, free=239068, buffers=272844, cached=8895316, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.61, five_min_avg=1.64, fifteen_min_avg=1.54, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd548560d016a61b5c74c2020626cb2ff57e7668bcfb4f5c04866853a8a997772', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 110857} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a3ea54cca2e752403bdc34ef6d9cdfefb4844063019c4355961501a281469968', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 110877} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9478340, partition_space_available_1k=71612144, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2527812, cpu_share=0.52, mem_res=1761824, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42352, cpu_share=0.0, mem_res=40088, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55840, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6775424, free=307232, buffers=272456, cached=8820520, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.73, five_min_avg=1.69, fifteen_min_avg=1.57, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cb644a7a4244c3f69077190e9b522ff79a75fab5f6568517d2c28abfdbb16d9b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 111470} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7e15dd6db9f110d8e82c28e517923494825eee55d2b5bfc71bc3ffed9dedccd8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 111493} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9487360, partition_space_available_1k=71603124, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2531488, cpu_share=0.08, mem_res=1760572, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42356, cpu_share=0.0, mem_res=40088, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55844, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6782664, free=290704, buffers=272476, cached=8829788, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.47, five_min_avg=1.62, fifteen_min_avg=1.55, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd53962ef29bd5724bf2e9880f8b288a8bee9cd0bb118c275bf0a91e438d60512', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 112032} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '774b917576e28f71b03bf6342fdb63d8b54993b61d6cc02a60641fddaca31ba5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 112054} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9492988, partition_space_available_1k=71597496, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2534708, cpu_share=0.06, mem_res=1760572, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42176, cpu_share=0.0, mem_res=39908, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55852, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6786216, free=281996, buffers=272528, cached=8834892, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.54, five_min_avg=1.62, fifteen_min_avg=1.55, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '54b1f767be9175403506f215ad0c969a6e829e14c6c97607b6f29ef98729395c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 112589} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9e1aa365f9f6d216a69d04e2658984ccbe2e2aac5691b7b93bb77436d73356b0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 112610} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9494172, partition_space_available_1k=71596312, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2538696, cpu_share=0.06, mem_res=1760636, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42232, cpu_share=0.0, mem_res=39964, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55856, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6794820, free=271468, buffers=272560, cached=8836784, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.85, five_min_avg=1.71, fifteen_min_avg=1.58, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2bc6d761c7d807e6eed0fc2d656a7a701ad21ba9f2f272622f1f3b2c4ce8e461', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 113160} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cefcf801f5dc558f0813b0afba105b1f38c93950b89db1c0cbf51d780eb2288a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 113182} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9500512, partition_space_available_1k=71589972, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2542392, cpu_share=0.06, mem_res=1760636, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42232, cpu_share=0.0, mem_res=39964, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55860, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6793020, free=267676, buffers=272604, cached=8842332, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.51, five_min_avg=1.65, fifteen_min_avg=1.57, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f583d5c881854a7d11124ad1e05c0467a5102630a566832b7ef4f86e61168d8c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 114010} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7325e249db82b55678407072bd80e4c7f41a70dabb1f76595c6bef775a5a52a2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 114041} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9507616, partition_space_available_1k=71582868, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2546584, cpu_share=0.14, mem_res=1760912, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42304, cpu_share=0.0, mem_res=40036, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55864, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6789600, free=261028, buffers=272748, cached=8852256, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=1.72, fifteen_min_avg=1.6, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ffde0f1185c2a01f1af284c3c9f1d9e021025c8123579921ea34c3780d3e8c43', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.38', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 114609} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '16965d626dde5767791035c3f52b3dd0dec5447b1ba3fe88d5c65806121425d3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'ea3c0ad52802fc219c662b2a9136e5c306ff1b85fbd5431b20d839bd518954c2', 'DetachKeys': '', 'Pid': 114629} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-2.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9509876, partition_space_available_1k=71580608, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2550184, cpu_share=0.06, mem_res=1760920, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=42304, cpu_share=0.0, mem_res=40036, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55868, cpu_share=0.01, mem_res=45128, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175632, used=6781132, free=258100, buffers=272796, cached=8863604, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.19, five_min_avg=1.56, fifteen_min_avg=1.55, cpu_share=6.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.38 - - [2025-08-28 04:19:56] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6756 0.001848