++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ CONTAINER_LOG_DIR=/var/log/contrail/database-nodemgr ++ mkdir -p /var/log/contrail/database-nodemgr ++ log_file=/var/log/contrail/database-nodemgr/console.log ++ touch /var/log/contrail/database-nodemgr/console.log ++ chmod 600 /var/log/contrail/database-nodemgr/console.log ++ exec +++ tee -a /var/log/contrail/database-nodemgr/console.log +++ date ++ echo 'INFO: =================== Thu Aug 28 02:08:15 AM UTC 2025 ===================' INFO: =================== Thu Aug 28 02:08:15 AM UTC 2025 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ ip route get 1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ cut -d / -f 1 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ ip addr show dev ens3 +++ awk '{print $2}' +++ grep 'inet ' +++ head -n 1 ++ DEFAULT_LOCAL_IP=10.0.0.41 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=256 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 256 ]] ++ [[ -z '' ]] +++ mount -t hugetlbfs +++ awk '/pagesize=2M/{print($3)}' +++ tail -n 1 ++ HUGE_PAGES_2MB_DIR= ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=openstack ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=rbac ++ AUTH_MODE=keystone ++ AUTH_PARAMS= ++ SSL_ENABLE=false ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_ALARM_ENABLE=True ++ ANALYTICS_SNMP_ENABLE=True ++ ANALYTICSDB_ENABLE=True ++ ANALYTICS_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICSDB_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_SNMP_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8081 ' +++ '[' -n '10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081 ' ']' +++ echo '10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081' ++ ANALYTICS_SERVERS='10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081' +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9042 ' +++ '[' -n '10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042 ' ']' +++ echo '10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042' ++ ANALYTICSDB_CQL_SERVERS='10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042' ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8086 ' +++ '[' -n '10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086 ' ']' +++ echo '10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086' ++ COLLECTOR_SERVERS='10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086' ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ CONFIGDB_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8082 ' +++ '[' -n '10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082 ' ']' +++ echo '10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082' ++ CONFIG_SERVERS='10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082' +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9161 ' +++ '[' -n '10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161 ' ']' +++ echo '10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161' ++ CONFIGDB_SERVERS='10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161' +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9041 ' +++ '[' -n '10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041 ' ']' +++ echo '10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041' ++ CONFIGDB_CQL_SERVERS='10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041' ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=false ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=false ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=false ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.20.0.25,10.20.0.17,10.20.0.254 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.20.0.25,10.20.0.17,10.20.0.254 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+=10.0.0.19:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+=10.0.0.38:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+=10.0.0.41:2181, +++ '[' -n 10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181, ']' +++ echo 10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181 ++ ZOOKEEPER_SERVERS=10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:2181 ' +++ '[' -n '10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181 ' ']' +++ echo '10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181' ++ ZOOKEEPER_SERVERS_SPACE_DELIM='10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181' ++ RABBITMQ_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+=10.0.0.19:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+=10.0.0.38:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+=10.0.0.41:5673, +++ '[' -n 10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673, ']' +++ echo 10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673 ++ RABBITMQ_SERVERS=10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=false ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:6379 ' +++ '[' -n '10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379 ' ']' +++ echo '10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379' ++ REDIS_SERVERS='10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379' ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=false ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9092 ' +++ '[' -n '10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092 ' ']' +++ echo '10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092' ++ KAFKA_SERVERS='10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092' ++ KAFKA_SSL_ENABLE=false ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=10.0.0.19 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=false ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=false ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=false ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ /v3 == \/\v\2\.\0 ]] ++ [[ openstack == \o\p\e\n\s\t\a\c\k ]] ++ AUTH_MODE=keystone ++ [[ keystone == \k\e\y\s\t\o\n\e ]] ++ AUTH_PARAMS='--admin_password contrail123' ++ AUTH_PARAMS+=' --admin_tenant_name admin' ++ AUTH_PARAMS+=' --admin_user admin' ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ keystone == \k\e\y\s\t\o\n\e ]] + cat + [[ http == \h\t\t\p\s ]] + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + [[ database == \t\o\r\a\g\e\n\t ]] + export NODEMGR_TYPE=contrail-database + NODEMGR_TYPE=contrail-database + NODEMGR_NAME=contrail-database-nodemgr ++ echo DATABASE ++ tr - _ + ntype=DATABASE + [[ DATABASE == \V\R\O\U\T\E\R ]] + [[ DATABASE == \D\A\T\A\B\A\S\E ]] + htype=ANALYTICSDB + [[ DATABASE == \C\O\N\T\R\O\L ]] ++ get_listen_ip_for_node ANALYTICSDB +++ find_my_ip_and_order_for_node ANALYTICSDB +++ local server_typ=ANALYTICSDB_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.19,10.0.0.38,10.0.0.41 +++ local servers=10.0.0.19,10.0.0.38,10.0.0.41 +++ local server_list= +++ IFS=, +++ read -ra server_list +++ cut -d ' ' -f 1 ++++ get_local_ips ++++ cat /proc/net/fib_trie ++++ grep -vi host ++++ tr '\n' , ++++ awk '/32 host/ { print f } {f=$2}' ++++ uniq ++++ sort +++ local local_ips=,10.0.0.41,10.20.0.254,127.0.0.1,172.17.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.19'\''))' +++ local server_ip=10.0.0.19 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.19 ]] +++ [[ ,10.0.0.41,10.20.0.254,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.19, ]] +++ (( ord+=1 )) +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.38'\''))' +++ local server_ip=10.0.0.38 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.38 ]] +++ [[ ,10.0.0.41,10.20.0.254,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.38, ]] +++ (( ord+=1 )) +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.41'\''))' +++ local server_ip=10.0.0.41 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.41 ]] +++ [[ ,10.0.0.41,10.20.0.254,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.41, ]] +++ echo 10.0.0.41 3 +++ return ++ local ip=10.0.0.41 ++ [[ -z 10.0.0.41 ]] ++ echo 10.0.0.41 + hostip=10.0.0.41 + introspect_ip=0.0.0.0 + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + mkdir -p /etc/contrail + cat ++ format_boolean false ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ echo False + '[' -n '' ']' + cat + add_ini_params_from_env DATABASE_NODEMGR /etc/contrail/contrail-database-nodemgr.conf + local service_name=DATABASE_NODEMGR + local cfg_path=/etc/contrail/contrail-database-nodemgr.conf + local delim=__ ++ set -o posix ++ set ++ grep '^DATABASE_NODEMGR__.*__.*=.*$' ++ sort ++ sed 's/^DATABASE_NODEMGR__//g' ++ cut -d = -f 1 + local vars=DEFAULTS__minimum_diskGB + local section= + for var in $vars + local var_name=DATABASE_NODEMGR__DEFAULTS__minimum_diskGB + local val=2 ++ echo DEFAULTS__minimum_diskGB ++ sed s/DASH/-/g + local var_unsubst=DEFAULTS__minimum_diskGB ++ echo DEFAULTS__minimum_diskGB ++ sed 's/^\(.*\)__.*$/\1/' + local var_section=DEFAULTS + [[ '' != \D\E\F\A\U\L\T\S ]] + echo '[DEFAULTS]' + section=DEFAULTS ++ echo DEFAULTS__minimum_diskGB ++ sed 's/.*__\(.*\)$/\1/' + local var_param=minimum_diskGB + echo 'minimum_diskGB = 2' + cat /etc/contrail/contrail-database-nodemgr.conf [DEFAULTS] http_server_ip=0.0.0.0 log_file=/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log log_level=SYS_DEBUG log_local=1 hostip=10.0.0.41 db_port=9042 db_jmx_port=7200 db_use_ssl=False [COLLECTOR] server_list=10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086 [SANDESH] introspect_ssl_enable=false sandesh_ssl_enable=false [DEFAULTS] minimum_diskGB = 2 + upgrade_old_logs contrail-database-nodemgr + local template=contrail-database-nodemgr ++ dirname /var/log/contrail/database-nodemgr + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/database-nodemgr + chmod 755 /var/log/contrail/database-nodemgr + mv -n '/var/log/contrail/*contrail-database-nodemgr.log*' /var/log/contrail/database-nodemgr/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/database-nodemgr + find /var/log/contrail/database-nodemgr -uid 0 -exec chown 1999:1999 '{}' + + exec /bin/sh -c '/usr/local/bin/contrail-nodemgr --nodetype=${NODEMGR_TYPE}' 08/28/2025 02:08:16.470 7f08c9fc2cc0 [contrail-database-nodemgr] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 08/28/2025 02:08:16.591 7f08c9fc2cc0 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 02:08:16.599 7f08c9fc2cc0 [contrail-database-nodemgr] [INFO]: SANDESH: INTROSPECT IS ON: 0.0.0.0:8103 08/28/2025 02:08:16.603 7f08c9fc2cc0 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 02:08:16.609 7f08c9fc2cc0 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 08/28/2025 02:08:16.609 7f08c9fc2cc0 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log] Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: CRITICAL:root:Exit data: {'ID': 'e9279ebf5abb89f3012450d8ae9590014abab0cf47511af3ec495fd454766757', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 29639} ERROR:contrail-database-nodemgr:Failed to connect to database by CQL and socket: Result: 1 INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-3.', _node_type='Database', _instance_id='0', _timestamp=1756346897353578, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Non-Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=[''], status='Down', description='none to Idle on EvStart')]), flag_infos=None, description='Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': '1e2740da627c37d27ae8d4e5e6e9b16fbdcbeda8a4f2b15f4300f055a1956546', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 29682} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:send_process_state_db: Sending UVE: NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-3.', _node_type='Database', _instance_id='0', _timestamp=1756346903574607, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=False, status=None, process_status=None, process_info=[ProcessInfo(process_name='cassandra', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-query-engine', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-database-nodemgr', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[])], disk_usage_info=None, description=None, all_core_file_list=None, build_info='{"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-08-28 00:14:38.756636", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10583-1.novalocal", "build-id" : "v0+nightly", "build-number" : "nightly.el9"}]}', process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=7011480, partition_space_available_1k=74079004, percentage_partition_space_used=9, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=[], build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1270480, cpu_share=0.0, mem_res=1262248, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=7052, cpu_share=0.0, mem_res=5560, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54588, cpu_share=0.0, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=4485476, free=4542996, buffers=220816, cached=6926348, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.18, five_min_avg=1.16, fifteen_min_avg=0.48, cpu_share=0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. deleted = False process_info = [ << process_name = cassandra process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-query-engine process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-database-nodemgr process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, ] build_info = {"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-08-28 00:14:38.756636", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10583-1.novalocal", "build-id" : "v0+nightly", "build-number" : "nightly.el9"}]} >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. disk_usage_info = { /dev/vda1 : << partition_type = ext4 partition_space_used_1k = 7011480 partition_space_available_1k = 74079004 percentage_partition_space_used = 9 >> } all_core_file_list = [ ] process_mem_cpu_usage = { cassandra : << mem_virt = 1270480 cpu_share = 0.0 mem_res = 1262248 >>contrail-query-engine : << mem_virt = 7052 cpu_share = 0.0 mem_res = 5560 >>contrail-database-nodemgr : << mem_virt = 54588 cpu_share = 0.0 mem_res = 45136 >> } system_mem_usage = << total = 16175636 used = 4485476 free = 4542996 buffers = 220816 cached = 6926348 node_type = database-node >> system_cpu_usage = << one_min_avg = 2.18 five_min_avg = 1.16 fifteen_min_avg = 0.48 cpu_share = 0 node_type = database-node >> >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.38:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.38:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346896622102 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-database-nodemgr:Session Event: TCP Connected INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-3. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001016 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[ClientInit] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 0 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Client: Event[EvSandeshCtrlMessageRecv] => State[ClientInit] -> State[Established] INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001438 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[Established] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 3 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '750c4ebe1b4d127feb5c90d18ad7ccca353a11a5f5e5255bc6b07b34d8528dcb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 34870} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-3.', _node_type='Database', _instance_id='0', _timestamp=1756346957649889, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=['10.0.0.38:8086'], status='Up', description='ClientInit to Established on EvSandeshCtrlMessageRecv')]), flag_infos=None, description='')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) CRITICAL:root:Exit data: {'ID': 'd0723eb89605e277dc74460315b05720ce0208307e2498b97d7917676ad3128f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 34892} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8337960, partition_space_available_1k=72752524, percentage_partition_space_used=10, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355844, cpu_share=0.33, mem_res=1336540, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=11736, cpu_share=0.0, mem_res=9768, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54624, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=5611424, free=2242172, buffers=264652, cached=8057388, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.9, five_min_avg=1.51, fifteen_min_avg=0.64, cpu_share=9.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a298c1dc2a48a59ce05aa804e0b864ed550145e338faeb400cc3d98adb194b54', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 36872} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2011b7a393b92850a895427c75c32854b37a09eef350b780e249b984ceec558c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 36892} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8366336, partition_space_available_1k=72724148, percentage_partition_space_used=10, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1369180, cpu_share=0.11, mem_res=1347332, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=11888, cpu_share=0.0, mem_res=9920, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54632, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6050008, free=1776068, buffers=266692, cached=8082868, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.29, five_min_avg=1.98, fifteen_min_avg=0.85, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7ac6f2e3fbbb346383b5a0fa0e9c6899f1a017b6c5308e63a7c6d698b55e18a5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 37430} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '604c3b5e0fe69b50fdd6763171d0afae07756d153932ed4371292bab13605c6c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 37451} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8370004, partition_space_available_1k=72720480, percentage_partition_space_used=10, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1364480, cpu_share=0.09, mem_res=1340200, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12028, cpu_share=0.0, mem_res=10060, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54640, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6121812, free=1699072, buffers=266744, cached=8088008, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.25, five_min_avg=1.83, fifteen_min_avg=0.87, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.41 - - [2025-08-28 02:11:39] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6758 0.001409 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.41 - - [2025-08-28 02:11:53] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6758 0.001175 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'de4da815c8bc3da74867269e01432b2185b2ded99149a53f684292342964e753', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 38527} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '750b27e7590c4187b654da6e60226727f5aa2e6b4a0b16cca60f4fb39d1a3391', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 38549} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8540708, partition_space_available_1k=72549776, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1375000, cpu_share=0.08, mem_res=1348076, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12204, cpu_share=0.0, mem_res=10232, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54644, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6138388, free=1497148, buffers=272044, cached=8268056, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.27, five_min_avg=1.62, fifteen_min_avg=0.86, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 02:13:08] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6758 0.001377 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '718c6d51a2ac3ae90717aa19fb65a62f7cf58c5cf8134a68e14d79696416f899', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 39517} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7efc0bd1ff0143cc10b25c4dcba7ceae28e4609cdfbba824934936adc7126bec', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 39537} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8546104, partition_space_available_1k=72544380, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1388360, cpu_share=0.11, mem_res=1357320, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12364, cpu_share=0.0, mem_res=10388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54664, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6177080, free=1452116, buffers=272196, cached=8274244, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.29, five_min_avg=1.57, fifteen_min_avg=0.89, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4dfd2bd0c17761772faf51cb48861d5a409c202b192368a12d7490e370bff92a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 40267} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3c92f18a26a9ad731cd77a705fbcfe01394505b91a14cf62bae02ae7ddfd7ac0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 40288} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8557880, partition_space_available_1k=72532604, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1406040, cpu_share=0.18, mem_res=1368596, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14520, cpu_share=0.0, mem_res=12492, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54660, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6252968, free=1363032, buffers=272332, cached=8287304, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.86, five_min_avg=1.66, fifteen_min_avg=0.97, cpu_share=6.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '05a865af5827cf220b2032e05c682c29726157a21a76b7832207931529caa8e0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 40885} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2dfda531e7a584e9e2d8a6fc0d87eafa989da02e55e484fea6826843b49434f3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 40906} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8565084, partition_space_available_1k=72525400, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1411200, cpu_share=0.08, mem_res=1369468, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13832, cpu_share=0.0, mem_res=11816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54672, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6260448, free=1349500, buffers=272456, cached=8293232, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.64, five_min_avg=1.66, fifteen_min_avg=1.01, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '42449a983d8b0f28d3c987136a3b5a4be931316e6824fc6013e7cff1879b5c00', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 41540} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6c8a7119c619b56a452f7071576f09768f5cdc25fdbdcf891d6c8f60e0c2f204', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 41561} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8577412, partition_space_available_1k=72513072, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1424928, cpu_share=0.1, mem_res=1376108, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13940, cpu_share=0.0, mem_res=11924, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54680, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6301952, free=1292780, buffers=272728, cached=8308176, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.11, five_min_avg=1.51, fifteen_min_avg=1.0, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '08df8f33f35a31c0257653e2b35032c19c3ec67da2710fa7e7e281ecc32b1d10', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 42260} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'db2dc133b31846663d51f7cfb01a2538dc7ea382cd86ccbe2d706b0f7f32e831', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 42281} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8593492, partition_space_available_1k=72496992, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1440204, cpu_share=0.13, mem_res=1384240, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14032, cpu_share=0.0, mem_res=12016, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54680, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6313020, free=1265976, buffers=272832, cached=8323808, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.18, five_min_avg=1.76, fifteen_min_avg=1.12, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '10eeddd05b84e43b766972a2b9c1d9b6849b355b4dd5144780b7c24237d19833', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 42817} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a855789ecbbc5dd1363a2470840ccbdfb17daea5988058289bd09cf251f6b7d3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 42838} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8598868, partition_space_available_1k=72491616, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1446224, cpu_share=0.06, mem_res=1386312, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14136, cpu_share=0.0, mem_res=12116, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54684, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6306228, free=1267428, buffers=272924, cached=8329056, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.22, five_min_avg=1.55, fifteen_min_avg=1.08, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8726c5f163a058d222a1074cabd981247795cae73cc8df3dbc01e62bdf79dde6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 43380} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f7d6ef00279ff3e7922eb11b69fd3590d7f82ce1a340dd9735fb9dee3a307747', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 43402} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8602968, partition_space_available_1k=72487516, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1451740, cpu_share=0.06, mem_res=1388808, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14148, cpu_share=0.0, mem_res=12128, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54696, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6307876, free=1261512, buffers=272992, cached=8333256, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.85, five_min_avg=1.4, fifteen_min_avg=1.06, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e897fc7ecb1241b10099afaf9bb9818524bf526c5250d5d1ae3f681dafec4a57', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 44132} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '69909b340f080c2f58436f301091dc601465f65f6b036815c1d19a7359929e57', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 44156} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8611792, partition_space_available_1k=72478692, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1467012, cpu_share=0.09, mem_res=1398992, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14200, cpu_share=0.0, mem_res=12180, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54700, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6335172, free=1223208, buffers=273056, cached=8344200, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.24, five_min_avg=1.4, fifteen_min_avg=1.08, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '983c204c3f3c8b2e2fe7f43ff4ae3f8f6e7012b7946f46521156207af7b3ec0f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 44794} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7fc4994ccdfa840450c69bd5e33a2d62dbe589ab60eb9138c644a13c38e761fa', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 44814} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8624268, partition_space_available_1k=72466216, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1476032, cpu_share=0.11, mem_res=1401792, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14076, cpu_share=0.0, mem_res=12056, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54700, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6341560, free=1205560, buffers=273148, cached=8355368, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=1.54, fifteen_min_avg=1.15, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '695d781ce56065c264a358663a73832a2d9a4a17931d06854e7f31dc50015454', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 45381} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c1fec29cbff25370f68a2b0db9c4d7ad32c8ee22df99abd37267681547676278', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 45403} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8638164, partition_space_available_1k=72452320, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1491092, cpu_share=0.11, mem_res=1408872, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14112, cpu_share=0.0, mem_res=12092, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54708, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6376688, free=1155000, buffers=273244, cached=8370704, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.95, five_min_avg=1.34, fifteen_min_avg=1.1, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e1acd58883e85dc957311fae28ddb425f3db361a48c27d6e58db910299561239', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 45978} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '4d04c2861eb46f118a334ef2f221130b4423bb84f3356471efd41b60c3a980ab', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 45999} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8645440, partition_space_available_1k=72445044, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1501656, cpu_share=0.07, mem_res=1414696, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14136, cpu_share=0.0, mem_res=12116, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54716, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6385948, free=1139312, buffers=273324, cached=8377052, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.75, five_min_avg=1.23, fifteen_min_avg=1.08, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '795f2821f6b1e607c795a0e0758aeaf1dc18a146e3ac57e737e3295c5e0f4c4b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 46572} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8c39acdc85051490d4126bfe9add3ea3d6a55e52b1980426e71e530ba454559e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 46595} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8656992, partition_space_available_1k=72433492, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1514680, cpu_share=0.09, mem_res=1421332, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14212, cpu_share=0.0, mem_res=12192, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54720, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6401088, free=1112156, buffers=273432, cached=8388960, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.64, five_min_avg=1.1, fifteen_min_avg=1.04, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '40819864ec2daea50a59b8f236d146d50f25838ce40603ee6eb63dd2b4f170e4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 47131} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '705daf1ba9e4e52c5f649462d070f6b6f462d0967a88c5fb6df3f49540f49606', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 47153} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8661892, partition_space_available_1k=72428592, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1522748, cpu_share=0.06, mem_res=1425612, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14212, cpu_share=0.0, mem_res=12192, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54724, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6409028, free=1099092, buffers=273476, cached=8394040, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.7, five_min_avg=1.03, fifteen_min_avg=1.02, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '62f4ac910a36a55346f70b17801569783c068f8d4f817c9199abcf8aba775a88', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 47681} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '30c4695e9a344a5a2f2df223d1b82920f7c6372a04bfe1fd512f8f460beef676', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 47704} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8666720, partition_space_available_1k=72423764, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1531236, cpu_share=0.06, mem_res=1430096, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14248, cpu_share=0.0, mem_res=12228, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54728, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6417948, free=1085156, buffers=273524, cached=8399008, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.59, five_min_avg=0.93, fifteen_min_avg=0.99, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bb5b9f60395c1ea46df039e9076c9dfad068a429f7a0ceca0e75ba4d309aa10b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 48237} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c5437c897233166402e90bbf65fba5df10eb079b1863b8033d8115cc53940791', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 48260} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8675788, partition_space_available_1k=72414696, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1540536, cpu_share=0.07, mem_res=1434936, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14192, cpu_share=0.0, mem_res=12172, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54736, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6428540, free=1064872, buffers=273600, cached=8408624, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.66, five_min_avg=0.89, fifteen_min_avg=0.97, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'aa595a377e81effa32dc2cf9ef0edb1a950b40dcf4824b5a53ef01106be9933f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 48847} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cc10fc21bb76fb27b9f12a27e914c6be53c445b8c7ea091b4a1ed899892b31b6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 48868} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8681272, partition_space_available_1k=72409212, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1547744, cpu_share=0.07, mem_res=1439044, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14196, cpu_share=0.0, mem_res=12176, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54736, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6434124, free=1053888, buffers=273648, cached=8413976, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.01, five_min_avg=0.94, fifteen_min_avg=0.98, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c079058ac4a7aa7da7c739f4f9181d52f0bd49e4ede0193b38e84bdfddedd9cb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 49465} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '145dc23443e951bfc9b4b4e3516b05765b29e53b5aa0f238c8c00b4d9a61e474', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 49486} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8689244, partition_space_available_1k=72401240, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1558248, cpu_share=0.07, mem_res=1444568, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14220, cpu_share=0.0, mem_res=12200, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54740, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6444784, free=1034712, buffers=273744, cached=8422396, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.69, five_min_avg=0.86, fifteen_min_avg=0.95, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '480c9d866f6ec44bca8aaabefa726e11e2ef0286a33b2c64318f02de78a04e80', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 50045} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8195fde57d3dfcb8b337c41594fa31f603da3b2a83a0c06fa3b48d3d8193d616', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 50068} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8719808, partition_space_available_1k=72370676, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1598152, cpu_share=0.14, mem_res=1454188, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14252, cpu_share=0.0, mem_res=12232, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54748, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6458980, free=989116, buffers=273824, cached=8453716, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.68, five_min_avg=0.81, fifteen_min_avg=0.93, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '00ddcd3a0e9a307853abcd77e18dd031ae8a7ef9bd553e06ac36aaba10bfb156', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 50707} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd7665cfd5d3733e4d9961c2690ee65211c960a82d3dd0760c564d5376d6eb45d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 50729} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8730764, partition_space_available_1k=72359720, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1602364, cpu_share=0.1, mem_res=1452924, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14304, cpu_share=0.0, mem_res=12284, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54756, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6461068, free=975252, buffers=273912, cached=8465404, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.21, five_min_avg=0.97, fifteen_min_avg=0.98, cpu_share=6.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e779cf6fccdee5e26be31ce0be30ba9cd7b2bfe9d99c9e39007f2accea75b524', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 51260} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c29bc1c64275dfb8f2467cbab8e3f9e73892ba83cf2386cbc7e214c0cedd3b20', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 51280} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8736020, partition_space_available_1k=72354464, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1605540, cpu_share=0.06, mem_res=1452924, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14304, cpu_share=0.0, mem_res=12284, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54756, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6461732, free=969304, buffers=274008, cached=8470592, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.7, five_min_avg=0.86, fifteen_min_avg=0.94, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8b18d9e18ef191cc80a6e5c4af25663f49e5e396c5b438b0bc461abb41eb1542', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 51818} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'eea373574e2084165e1cdaba87f40bcc0f540a38d8640b42a9d3d4a7aacf4287', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 51839} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8740084, partition_space_available_1k=72350400, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1608840, cpu_share=0.06, mem_res=1453324, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14248, cpu_share=0.0, mem_res=12228, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54760, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6460932, free=965964, buffers=274088, cached=8474652, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.64, five_min_avg=0.83, fifteen_min_avg=0.92, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0f54d1ca1cfa47657f916fa685977fef694e9e3c3695ac9377af55bec71a1f91', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 52415} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'bfb03b4ed36a209cb83166adf4ecfbfb08ea7f217774c1e049036633dd49f0c9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 52437} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8750556, partition_space_available_1k=72339928, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1615204, cpu_share=0.09, mem_res=1453352, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14272, cpu_share=0.0, mem_res=12252, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54764, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6467808, free=947840, buffers=274168, cached=8485820, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.27, five_min_avg=1.45, fifteen_min_avg=1.12, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '703a9ee7e9dc8c16429060dcc00d7f86b424a181541514c246586325a032170f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 52966} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a9dbd8f014c056fc269da5a0a1551a6d11d8c3548ce38d943e828f0e4c77016f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 52987} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8755596, partition_space_available_1k=72334888, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1619136, cpu_share=0.06, mem_res=1453776, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14316, cpu_share=0.0, mem_res=12296, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54768, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6467144, free=943428, buffers=274228, cached=8490836, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.53, five_min_avg=1.27, fifteen_min_avg=1.08, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'af4f116b9d4ed938cefeb35b38eb23223105db13ba43778f6d078cd7b749fe26', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 53537} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'bf33f9a4b76d5c6313d3244e232b37aa26fb526d2fb40339d07e73f7200ac6d6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 53568} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8761748, partition_space_available_1k=72328736, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1623336, cpu_share=0.07, mem_res=1453792, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14376, cpu_share=0.0, mem_res=12356, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54776, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6474444, free=928864, buffers=274300, cached=8498028, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.0, five_min_avg=1.16, fifteen_min_avg=1.06, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c6e166adb4730c7bdee39e3e29e7edfaad6328415d6baf087812e4a577dc97ba', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 54118} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7addb2c5c2abdc83cb065f8a2915417a15800849b12aa052ae74917e47ce5b66', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 54140} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8769208, partition_space_available_1k=72321276, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1628332, cpu_share=0.07, mem_res=1454536, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14468, cpu_share=0.0, mem_res=12448, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54776, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6478636, free=917908, buffers=274356, cached=8504736, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.42, five_min_avg=1.23, fifteen_min_avg=1.08, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '378282c00024559379bc29a684d8733090c1780a36e44e7cbc2358dd7399fd11', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 54725} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '75b0edaaa03c9b5ca7be8b46241bb8b2875794bcdfb0a55b9bf84abae11b3396', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 54748} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8790364, partition_space_available_1k=72300120, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1626964, cpu_share=0.1, mem_res=1435944, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14472, cpu_share=0.0, mem_res=12448, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54784, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6465012, free=909580, buffers=274448, cached=8526596, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=1.37, fifteen_min_avg=1.14, cpu_share=5.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0049778ef5703157549cc21da3bef94f6d0dc1130a00c65a88a3eabb82a4238e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 55291} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '93b8e48de4e9b35dcce8c77be5674419917c29d24cd01788d8266902ef94149e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 55313} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8794684, partition_space_available_1k=72295800, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1629148, cpu_share=0.06, mem_res=1435152, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14344, cpu_share=0.0, mem_res=12320, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54788, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6462876, free=907072, buffers=274516, cached=8531172, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.13, five_min_avg=1.26, fifteen_min_avg=1.12, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'da58106e108d2b4e676d6ad928a1c6446f1be5c2f5ce2391fdb0f9ed2ee565e4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 55856} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3e935fd79dba69233b718b690f5f4a7b6c0635b3380c28955343d143a4480357', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 55878} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8802360, partition_space_available_1k=72288124, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1634056, cpu_share=0.07, mem_res=1435520, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14416, cpu_share=0.0, mem_res=12392, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54796, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6492016, free=869212, buffers=274572, cached=8539836, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.93, five_min_avg=1.16, fifteen_min_avg=1.09, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1ecbe057c6f62b2e775e437fb62f668c9ac227247cf0f30c7d028635e0081aa9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 56449} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '72f5e37fc6085d91ff8b2e67aa108fd94e97e6eaf0168d4444c4655e5a38fe0a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 56471} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8811316, partition_space_available_1k=72279168, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1639228, cpu_share=0.08, mem_res=1435532, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14416, cpu_share=0.0, mem_res=12392, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54796, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6492272, free=860748, buffers=274656, cached=8547960, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.83, five_min_avg=1.79, fifteen_min_avg=1.32, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ca4ae04cc78028400a172418c77c0c5f807a39fd2eb5a23c48d1bfbb56a5e574', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 57003} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9e38d1d253c3d2a3f8c46a65f10a18f1236bfc51977df8171576685f8c7a111d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 57025} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8816424, partition_space_available_1k=72274060, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1642724, cpu_share=0.06, mem_res=1435536, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14452, cpu_share=0.0, mem_res=12428, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54800, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6494468, free=853380, buffers=274712, cached=8553076, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.32, five_min_avg=1.55, fifteen_min_avg=1.26, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '431db6344715ad4de450e4ebfae52a903192739ee98a19a9a2df4d9edaad1737', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 57567} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1aa3ded2557ad5c5c2409a69fe2bfb492b0f4334412e1e844716bf4d77ee3f9b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 57588} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8820512, partition_space_available_1k=72269972, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1645952, cpu_share=0.06, mem_res=1435692, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14452, cpu_share=0.0, mem_res=12428, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54808, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6497368, free=846044, buffers=274748, cached=8557476, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.85, five_min_avg=1.37, fifteen_min_avg=1.22, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0f5743e673c9c43b411c71dd71e1f7a35e4b944d2d5bbcbf84298515b4c979c5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 58158} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'aaeed7f75fa625f937a6b3079b2b880c56295ed610cfea1c3dfae7246835f6b2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 58180} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8832660, partition_space_available_1k=72257824, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1652032, cpu_share=0.1, mem_res=1435564, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14452, cpu_share=0.0, mem_res=12428, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54812, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6502148, free=828888, buffers=274784, cached=8569816, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.18, five_min_avg=1.37, fifteen_min_avg=1.22, cpu_share=5.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2f82e32aef76a4d31f75c7671181616a57c14cc2d8bef04e75134e549e4de01c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 58715} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c1123eed4e41ba2776944805b431a95f529231e19efeba3fd7cb689e8654cd49', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 58736} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8837204, partition_space_available_1k=72253280, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1655704, cpu_share=0.06, mem_res=1435768, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14380, cpu_share=0.0, mem_res=12356, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54816, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6502916, free=823556, buffers=274832, cached=8574332, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.72, five_min_avg=1.2, fifteen_min_avg=1.18, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '119d243483f8ab1a8b2efea8bb8a74f413fad7845c9838c8e447a8baaadb32ac', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 59269} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3db0a7b68e4cabf396b655664064d50500d5431312cce415ed0cc045a7d9e534', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 59289} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8841984, partition_space_available_1k=72248500, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1661120, cpu_share=0.06, mem_res=1437640, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14400, cpu_share=0.0, mem_res=12376, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54820, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6507284, free=814084, buffers=274908, cached=8579360, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.59, five_min_avg=1.07, fifteen_min_avg=1.13, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '88323c65bfdf8085a2cefd12e1682139bde0c431f297b4a29c261fdf2fefa327', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 59827} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '57455641c6ef914343e53b77eda52b78b94f5872f9a54b8b36dfe9c4217f7142', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 59849} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8849568, partition_space_available_1k=72240916, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1666356, cpu_share=0.07, mem_res=1438744, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14408, cpu_share=0.0, mem_res=12384, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54828, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6515948, free=796084, buffers=274976, cached=8588628, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.45, five_min_avg=0.95, fifteen_min_avg=1.08, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8388bab6a055bd488a9bc1c7df2986bdb26443d7778b112e35d241ab7f0b1bd2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 60395} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2058bac25e72b33c1f2faf364eac8366bb76b4dc7bc6ce414d08886f23317e3e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 60415} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8855800, partition_space_available_1k=72234684, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1672092, cpu_share=0.07, mem_res=1440936, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14412, cpu_share=0.0, mem_res=12388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54832, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6521440, free=785592, buffers=275060, cached=8593544, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.98, five_min_avg=1.02, fifteen_min_avg=1.1, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '724c292425df49d0f06e43ba042954618a4ce13eb649d9fcdd0e6b2104bf4eb3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 61002} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '815f4ef57c17017d80db28fca0e584b0814a6cb05188d3271a882f48fb8d655c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 61023} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8864780, partition_space_available_1k=72225704, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1679796, cpu_share=0.08, mem_res=1442584, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14460, cpu_share=0.0, mem_res=12436, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54836, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6525064, free=772668, buffers=275184, cached=8602720, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.9, five_min_avg=0.98, fifteen_min_avg=1.08, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '88efbd921a4a7b83ff1108cb62758a17a5275467b1fcb2109ce299c5371cd3bc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 61616} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6791b725d346b0810100d281261ac49ee9dcfa11182cca3818612e473a3e4b41', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 61637} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8874680, partition_space_available_1k=72215804, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1692776, cpu_share=0.09, mem_res=1449360, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14464, cpu_share=0.0, mem_res=12440, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54840, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6538148, free=748728, buffers=275304, cached=8613456, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.79, five_min_avg=0.94, fifteen_min_avg=1.06, cpu_share=5.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4c1082d7a529dcb56a3f2b3df9cafd43aa84c1d62336547b855c68e6b48652a2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 62202} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b5ee2ef70e057d396c8145a1ba914c006348a64c4ed448be29a424aed9d56d95', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 62223} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8880716, partition_space_available_1k=72209768, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1697540, cpu_share=0.07, mem_res=1450392, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14392, cpu_share=0.0, mem_res=12368, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54844, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6543004, free=737864, buffers=275392, cached=8619376, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.69, five_min_avg=0.88, fifteen_min_avg=1.03, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'eec8f8e648649b88532cc1b44d9c748f27f8ecf8e09f4966201aaf8dd4a0bd5d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 62786} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0ef64aba2ed4434330d6c4540f430feb2483a3bf6443e122b915fb0529756235', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 62807} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8886496, partition_space_available_1k=72203988, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1702452, cpu_share=0.06, mem_res=1451636, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14408, cpu_share=0.0, mem_res=12384, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54848, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6545772, free=729164, buffers=275440, cached=8625260, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.16, five_min_avg=1.05, fifteen_min_avg=1.08, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bbe1e3703d86b19ee6e9ffd744ccdef16b11880d0d35a6f25bf51f479d221098', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 63385} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '64dfd0dc491ff0970214db402551388465d2958127cc6c96bea4ea9a205369e8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 63406} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8919688, partition_space_available_1k=72170796, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1727128, cpu_share=0.14, mem_res=1444932, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14512, cpu_share=0.0, mem_res=12488, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54856, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6541512, free=699112, buffers=275540, cached=8659472, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.14, five_min_avg=1.08, fifteen_min_avg=1.09, cpu_share=5.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd7e6b041daf4d9a56c7839e1b5a76c409b53c36d7f2ddf3877802a6446aa1c87', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 63943} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e4d154242b0e0a0d2dee5c40ab9fbebe64b49b06efdd37164e9be226e0da3645', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 63964} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8924448, partition_space_available_1k=72166036, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1729636, cpu_share=0.06, mem_res=1444144, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14608, cpu_share=0.0, mem_res=12584, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54864, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6540800, free=695000, buffers=275580, cached=8664256, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.68, five_min_avg=0.95, fifteen_min_avg=1.04, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'daf87db4f84be8c9e276b2be73fd9e553914b02e8195d202c5af83b84090021b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 64495} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9f68aa1735cb49ed95e709d52cff79518e351f57c2f35aea7a593c6e70bb09c9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 64516} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8928452, partition_space_available_1k=72162032, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1732596, cpu_share=0.06, mem_res=1444184, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14616, cpu_share=0.0, mem_res=12592, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54868, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6543568, free=687988, buffers=275592, cached=8668488, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.29, five_min_avg=1.05, fifteen_min_avg=1.06, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '42980ea144f0daabac0f5ccef5a8353f784fe4250ca44a82f69f518b8a424ff3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 65067} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '259e98c3e5b255aa0458c95a93f7a0f0a799dd728d749915aa2f0e8197e9dbe1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 65088} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8934612, partition_space_available_1k=72155872, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1737100, cpu_share=0.07, mem_res=1444116, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14616, cpu_share=0.0, mem_res=12592, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54872, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6552708, free=671296, buffers=275680, cached=8675952, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.12, five_min_avg=1.01, fifteen_min_avg=1.05, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8545e4a04f06cbb3cf1307065cc310b9b88a4e3e58359c89c96856c151a4322a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 65635} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6d3b5e1d3904bcefe76acff90e45196dab23aa0ba52d334c2b02ff8a891a7018', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 65657} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8941548, partition_space_available_1k=72148936, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1740840, cpu_share=0.07, mem_res=1444052, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14544, cpu_share=0.0, mem_res=12520, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54876, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6550684, free=667336, buffers=275756, cached=8681860, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.28, five_min_avg=1.1, fifteen_min_avg=1.08, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0488d7b3e913c4f3840af1cb584766b1fc961c7f38912b40da16e19cbab5d336', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 66199} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '49aee7d563a622cbe13358d635330631970fd9f45153632ee796e620600da5f8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 66221} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8946688, partition_space_available_1k=72143796, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1745156, cpu_share=0.06, mem_res=1444120, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14560, cpu_share=0.0, mem_res=12532, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54880, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6551120, free=661544, buffers=275832, cached=8687140, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.83, five_min_avg=0.99, fifteen_min_avg=1.04, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2c11afcb763b7326f7198a32db5155c76f67a17e3dceb06eae3c09db748aebc2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 66764} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1488f5e747261155ee3d6495006a8c21013d199619b242330957d9801d0b3b2e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 66786} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8951096, partition_space_available_1k=72139388, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1748448, cpu_share=0.06, mem_res=1444060, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14576, cpu_share=0.0, mem_res=12548, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54888, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6551528, free=656328, buffers=275876, cached=8691904, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.89, five_min_avg=0.96, fifteen_min_avg=1.02, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '330867a206ad3ae868bac1674b7fc256a36ee58f1c7d49cd2c81d360d8dce4cd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 67350} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '886174facf16f4a682eb53e9f19fa0ba39dbaf745d023dbc35df041eecbfc096', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 67371} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8959128, partition_space_available_1k=72131356, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1754024, cpu_share=0.09, mem_res=1444260, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14608, cpu_share=0.0, mem_res=12580, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54888, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6556904, free=642604, buffers=275932, cached=8700196, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.16, five_min_avg=1.03, fifteen_min_avg=1.04, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '91e88488b4f44bb2076f72458863c736040906d6374cedb0a55def2cdd9f9bf8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 67896} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '675f3ba28a83afb8f22b974a95b936981a00807abf4595f1f1a052db743b9b53', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 67917} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8964116, partition_space_available_1k=72126368, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1757516, cpu_share=0.06, mem_res=1444196, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14648, cpu_share=0.0, mem_res=12620, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54896, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6559420, free=635048, buffers=275976, cached=8705192, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.81, five_min_avg=0.95, fifteen_min_avg=1.01, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '91795a9ab661abff3acedb7611febc7f128a22cda2fb5689c0fe0a780e384dc7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 68450} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '06cff74272d24c2017463176ddcf6d366ce3fe8d9c9678f3a80a952e061aca01', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 68474} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8968608, partition_space_available_1k=72121876, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1761828, cpu_share=0.06, mem_res=1445468, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14656, cpu_share=0.0, mem_res=12628, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54900, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6565096, free=624348, buffers=276048, cached=8710144, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.92, five_min_avg=0.98, fifteen_min_avg=1.01, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6917200f087cc24c8b1a0a4656d3150c8273dae82e5ee142ba5d2f3d3ade9d14', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 69027} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '85f0f7464d7ea46b02b5478b76f5796cbd1a894dfdfcd21eaba973643bdc2e5f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 69047} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8974076, partition_space_available_1k=72116408, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1769168, cpu_share=0.07, mem_res=1448808, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14604, cpu_share=0.0, mem_res=12576, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54908, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6568716, free=614736, buffers=276080, cached=8716104, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.83, five_min_avg=0.93, fifteen_min_avg=0.99, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '28d4cec16530a50bff223c40ca5bfc125f5d12eb305efa44e346b5c17df0b04b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 69672} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '285d810cbc2a93ee2560af268fdc96e1b98f6073176d76e764070bd81b897a7b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 69693} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=8989320, partition_space_available_1k=72101164, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1799520, cpu_share=0.14, mem_res=1460492, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14624, cpu_share=0.0, mem_res=12596, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54908, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6590644, free=567052, buffers=276168, cached=8741772, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.29, five_min_avg=1.07, fifteen_min_avg=1.04, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5e413db152d52ffe038cef895be022649671caa283751354dc2ee710cf0e6acc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 70339} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1c1edcf4a9cea17345f6d5e7ccfdbd6619fdb9f8a4e2767e7e7c8f897a7535b1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 70360} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9015164, partition_space_available_1k=72075320, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1800776, cpu_share=0.13, mem_res=1453060, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14664, cpu_share=0.0, mem_res=12636, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54916, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6594240, free=546388, buffers=276244, cached=8758764, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.61, five_min_avg=1.71, fifteen_min_avg=1.25, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5c624ca58e7aabcf776713ca49a3182cd183c056e961b786c30d097515fa3b29', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 70901} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'bb77fba2b44d0a4f23407b456d4cd1bdd927c08bffb18d1f1dbbe752db5b79a1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 70924} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9021116, partition_space_available_1k=72069368, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1804276, cpu_share=0.07, mem_res=1453040, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14668, cpu_share=0.0, mem_res=12640, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54920, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6588628, free=546528, buffers=276392, cached=8764088, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.94, five_min_avg=1.57, fifteen_min_avg=1.23, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8de05699e238199e00d86b5b489dfe7f962c39623a95d81e7b28c9a57fb09bca', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 71457} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f9cf4808ee359e3ac724a3401f51e7e9dcac69c57ba33a56ffbea045aa518f12', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 71477} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9027508, partition_space_available_1k=72062976, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1808632, cpu_share=0.06, mem_res=1453180, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14720, cpu_share=0.0, mem_res=12692, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54924, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6591684, free=536800, buffers=276500, cached=8770652, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.07, five_min_avg=1.37, fifteen_min_avg=1.18, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ea76c69575930bdd7db7d5ee7bced70983c7f4973a0380f4ac253b56796e60d4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 72060} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c6091e9b4ab2a4b462253a1ec74518dc1b35b15f262403fdb668ec8770ec05ee', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 72081} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9036632, partition_space_available_1k=72053852, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1809808, cpu_share=0.08, mem_res=1448260, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14724, cpu_share=0.0, mem_res=12696, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54932, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6592632, free=526012, buffers=276780, cached=8780212, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.88, five_min_avg=1.24, fifteen_min_avg=1.15, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '86f7b508bbddbb8785848059109373fc37df8fa95686233edf6afaeca5a06305', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 72633} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e84fc2a9b316da8a2ad31381a2c0691879d478ea149b61455aa4e87dd13476a0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 72655} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9035864, partition_space_available_1k=72054620, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1812164, cpu_share=0.05, mem_res=1447860, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14608, cpu_share=0.0, mem_res=12580, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54932, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6598892, free=520468, buffers=276872, cached=8779404, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.71, five_min_avg=1.14, fifteen_min_avg=1.12, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1c4fc7cc1f8f5772d956d8723f5613f31c161463878b00dbafd7ddf0c3e4c9e7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 73189} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f1450399ba66bce6ebe0eb780c7cd10e11599946dd534419ae35f5945b709b1e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 73210} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9041516, partition_space_available_1k=72048968, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1816692, cpu_share=0.07, mem_res=1447672, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14612, cpu_share=0.0, mem_res=12584, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54944, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6582772, free=530672, buffers=276940, cached=8785252, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.68, five_min_avg=1.05, fifteen_min_avg=1.09, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e618d4a2f89a05c442a76eb6989924797729e37a8ee19602968c08defe848cfb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 73778} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a8bd394bc0180bbb4a203843b0b37ab140fbacfb64839b6695512141af20fa19', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 73801} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9049292, partition_space_available_1k=72041192, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1821876, cpu_share=0.08, mem_res=1447604, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14612, cpu_share=0.0, mem_res=12584, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54952, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6588888, free=516656, buffers=276996, cached=8793096, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.03, five_min_avg=1.08, fifteen_min_avg=1.09, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '08e8667eee8dc98db07690b19725fe836c131fd03fee6b3a787d177b4afb1f90', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 74336} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '50fb67866bada5e167b196bdb049b5a06e40d810fb8ecd01d21957631c3f35e2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 74356} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9053964, partition_space_available_1k=72036520, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1825052, cpu_share=0.06, mem_res=1447760, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14620, cpu_share=0.0, mem_res=12592, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54952, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6592276, free=508472, buffers=277036, cached=8797852, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.74, five_min_avg=0.99, fifteen_min_avg=1.06, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '201f14833f7b598c4f16914d54eacd6ef477df1ac6f3086fb9aec2b81e764d8e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 74903} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6fdd50ca4133c3226949688cbb728af0b7f9bfc4efdcd97a7cae08cbe4837740', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 74924} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9087676, partition_space_available_1k=72002808, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1854748, cpu_share=0.13, mem_res=1443504, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14624, cpu_share=0.0, mem_res=12596, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54956, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6589988, free=476044, buffers=277120, cached=8832484, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.72, five_min_avg=0.93, fifteen_min_avg=1.03, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f7d9c8f5b35b94a7aa0c3731613602cc3f9e1297759cb6984d000756bf743d15', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 75488} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b193508b4e06a8cb33fe81983959a9f9f43ff33100623a91cb657c35365e9088', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 75508} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9095268, partition_space_available_1k=71995216, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1859052, cpu_share=0.07, mem_res=1442788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14628, cpu_share=0.0, mem_res=12600, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54960, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6596536, free=461212, buffers=277188, cached=8840700, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.86, five_min_avg=0.93, fifteen_min_avg=1.02, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2cc317c50e91a570633dd3f684af857d074849a3098581e03c8049b9907a7f22', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 76097} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c672254ddf975aa5bdef32293537926fa9e1548a5bfeb1cd09b5ae64f85dff8a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 76118} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9103100, partition_space_available_1k=71987384, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1864516, cpu_share=0.08, mem_res=1442876, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14604, cpu_share=0.0, mem_res=12576, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54968, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6589712, free=459372, buffers=277272, cached=8849280, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.64, five_min_avg=1.44, fifteen_min_avg=1.19, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5d07c1eb8d3a108ebb38844cea13b7b123428972649bac217e9ff05a88032792', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 76684} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '29c1cd0a55800cb35330ae0230f795c6570a3809fd4de7272d5265ade72226b6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 76704} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9105896, partition_space_available_1k=71984588, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1869536, cpu_share=0.08, mem_res=1442740, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14616, cpu_share=0.0, mem_res=12588, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54968, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6588872, free=457988, buffers=277328, cached=8851448, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.44, five_min_avg=1.57, fifteen_min_avg=1.25, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '12fccf4414b89712e79afc7766cadb36fb61074ea7b905fe8844391a3ac54d35', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 77248} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '518f3e964e1cdee8edefd4a0329aa8b9b0012cec824a0936e03ace49c84eca48', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 77269} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9111400, partition_space_available_1k=71979084, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1873404, cpu_share=0.05, mem_res=1442888, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14672, cpu_share=0.0, mem_res=12644, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54976, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6594020, free=447404, buffers=277384, cached=8856828, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.53, five_min_avg=1.45, fifteen_min_avg=1.23, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '587f3cb6c11dc145ecf71ad72b621ad38ca0ef99fce8b6bdecaf8b452a1fa35d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 77802} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e976e6caa4bfb50497173c63e92237092425f95577407936d7ba322ca912eed8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 77824} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9115012, partition_space_available_1k=71975472, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1876164, cpu_share=0.06, mem_res=1442888, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14676, cpu_share=0.0, mem_res=12648, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54980, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6593524, free=443984, buffers=277440, cached=8860688, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.18, five_min_avg=1.34, fifteen_min_avg=1.2, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7b18326d34b57c276ce2473838cce14822a6c9d1c0446d5204fd5cbbcdc4e832', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 78359} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '05c4a6cedeea496e81b6889a3a97f8bec23005924928ebaec3ebe01a9dff26cb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 78380} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9119064, partition_space_available_1k=71971420, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1879340, cpu_share=0.06, mem_res=1442952, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14720, cpu_share=0.0, mem_res=12688, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54988, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6597792, free=435224, buffers=277484, cached=8865136, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.9, five_min_avg=1.2, fifteen_min_avg=1.16, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '80f13ff7086faf6c671248dc12d87f33cdebda05c56b99a89e1dd0ab9f22f517', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 79056} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1b883d93ae0336fb30f2e438061992260f773582a30643b9db4a0fdc03612c0a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 79077} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9124216, partition_space_available_1k=71966268, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1885184, cpu_share=0.09, mem_res=1442952, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14724, cpu_share=0.0, mem_res=12692, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54988, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6601076, free=426184, buffers=277540, cached=8870836, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.57, five_min_avg=1.31, fifteen_min_avg=1.2, cpu_share=5.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4407a7b115fcf398cf7364eeef5866486ac29b026c7aaad9f5066c40ef2ed5a4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 79664} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a9a84bae81fc36ad108da12e023bbffda2ddfb1b4374394cfd158e84f9709d8c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 79685} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9131912, partition_space_available_1k=71958572, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1889860, cpu_share=0.07, mem_res=1443080, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14620, cpu_share=0.0, mem_res=12588, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54996, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6601048, free=418876, buffers=277584, cached=8878128, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.06, five_min_avg=1.18, fifteen_min_avg=1.16, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '52cfde1f2b5fefec48f0f7bbac82dcc815dc2c6eaa6749572caa2674c7001f5a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 80224} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a861d5ca1e2567bbdcebab0d65a019f073819c0bb0327827a0c01dea8738f8cb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 80247} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9137516, partition_space_available_1k=71952968, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1894744, cpu_share=0.06, mem_res=1444036, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14680, cpu_share=0.0, mem_res=12648, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55004, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6602968, free=411176, buffers=277672, cached=8883820, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.98, five_min_avg=1.12, fifteen_min_avg=1.14, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8676f724a9ef0b851fb03443d220fe827d828e806baba22738f49b5d0e44fcc6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 80777} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9b6fa7b69a23f927c9eae5c4f0ad6f698cedebfb72ecfa623f9f3bca5782566d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 80798} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9137404, partition_space_available_1k=71953080, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1900708, cpu_share=0.06, mem_res=1446456, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14680, cpu_share=0.0, mem_res=12648, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55008, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6607312, free=406972, buffers=277712, cached=8883640, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.67, five_min_avg=1.01, fifteen_min_avg=1.09, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f98456f2256d24183c23a601e93bedce171a8f7fd19ba56ae5b879709abe895e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 81337} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9bc82badeb75f7b6dec69de00fe70c884fb95aadb1f43c1989ad8bbab47eb31d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 81359} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9142392, partition_space_available_1k=71948092, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1906480, cpu_share=0.06, mem_res=1448088, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14680, cpu_share=0.0, mem_res=12648, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55012, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6606656, free=402500, buffers=277756, cached=8888724, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.97, five_min_avg=1.01, fifteen_min_avg=1.09, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5c2a601b402f4b79d9dfb7aab6522f57ad64ea9eaf91fbec2f4f52436fbebb81', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 81927} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c61998a0ea7011320d781421d74c2f70753b6efd443a98369e1d59b441d38c76', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 81949} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9150620, partition_space_available_1k=71939864, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1919660, cpu_share=0.09, mem_res=1455376, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14700, cpu_share=0.0, mem_res=12668, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55012, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6623380, free=376916, buffers=277860, cached=8897480, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.02, five_min_avg=1.0, fifteen_min_avg=1.08, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a970cfb749788f509ffa4698a152a7474e03507c2ccb80e56bd44703694d248d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 82512} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2fe779544b9d1f5074f7764a448e31abd42ba14aef786cbb8eb7b158fe96805d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 82532} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9159844, partition_space_available_1k=71930640, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1925776, cpu_share=0.09, mem_res=1455652, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14700, cpu_share=0.0, mem_res=12668, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55020, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6625772, free=364968, buffers=277956, cached=8906940, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.63, five_min_avg=1.13, fifteen_min_avg=1.11, cpu_share=5.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '508a5c512e6570a86eff91594a17caf8bdfdb702635cca86a0c1dfef5605b0e2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 83058} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b458f67003601f012dbc830536e2c0d9049d0e6b2e70e0f915a7b314522be964', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 83079} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9165556, partition_space_available_1k=71924928, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1930564, cpu_share=0.06, mem_res=1456384, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14636, cpu_share=0.0, mem_res=12604, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55024, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6591452, free=393472, buffers=278004, cached=8912708, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.08, five_min_avg=1.05, fifteen_min_avg=1.08, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a3f234a012b125d73867cdd71ba9a3e0fabb50beebe13570751f429c174edce2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 83616} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'faa7a17b37dc1aafb5e9b014fc6a99b9f3d1524215c688425195f1ca10ce2a26', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 83636} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9170092, partition_space_available_1k=71920392, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1938108, cpu_share=0.06, mem_res=1459788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14692, cpu_share=0.0, mem_res=12660, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55028, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6642868, free=304940, buffers=278072, cached=8949756, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.6, five_min_avg=1.16, fifteen_min_avg=1.12, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b658df6dc1efbc14f8c3f868dc59e04cd124b25563036b0ca89d7c476c10c2a5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 84217} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '36f2d51871525445784e1159e8080b7ae4091f5d2b81e99fd919acf9b93909dd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 84241} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9181744, partition_space_available_1k=71908740, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2041896, cpu_share=0.37, mem_res=1551840, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14696, cpu_share=0.0, mem_res=12664, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55024, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6700304, free=304912, buffers=277752, cached=8892668, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.23, five_min_avg=1.15, fifteen_min_avg=1.12, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '87fbc8af572538c77fed205fc8662d1e0c7c70d077c7ff98f58feb3d1cf44877', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 84779} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ed37b04829171a8e38b18c20d09458b642f9c245bdea247a25362d59b73d5a36', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 84802} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9186104, partition_space_available_1k=71904380, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2043560, cpu_share=0.06, mem_res=1550572, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14748, cpu_share=0.0, mem_res=12716, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55032, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6716988, free=283960, buffers=277780, cached=8896908, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.97, five_min_avg=1.13, fifteen_min_avg=1.11, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '87c0d2720404691ad33c6ebf45d0caf8b217af964707c223ece613d5856f0f5e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 85363} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '77883a13284488fbe68221a29cd4ca367e246ada787619214f4bc91677f32145', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 85384} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9192256, partition_space_available_1k=71898228, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2048272, cpu_share=0.07, mem_res=1550636, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14748, cpu_share=0.0, mem_res=12716, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55036, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6723388, free=269820, buffers=277808, cached=8904620, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.58, five_min_avg=1.22, fifteen_min_avg=1.14, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7629d69e093810a0f6d3d077f971aedb88eb008ca60301c11856480d185f4c50', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 85912} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'dc0906ba5dd78aa6823aef6a48dede3b2801706eed925ff028a9fe19e27ebbf4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 85933} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9197756, partition_space_available_1k=71892728, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2053072, cpu_share=0.06, mem_res=1552088, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14748, cpu_share=0.0, mem_res=12716, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55036, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6731592, free=256604, buffers=277844, cached=8909596, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.98, five_min_avg=1.1, fifteen_min_avg=1.1, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '96afcf7429315c84bfd5bab6da428e9655c6538fbf10d2b6847b6bb9694881da', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 86635} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '893535ddc764ea71a835d6d6d141f38aedfce51a99da454f161cd8aa8e06789e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 86655} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9203984, partition_space_available_1k=71886500, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2057536, cpu_share=0.07, mem_res=1552580, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14636, cpu_share=0.0, mem_res=12604, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55044, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6734604, free=245892, buffers=277900, cached=8917240, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.03, five_min_avg=1.08, fifteen_min_avg=1.09, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '83c2d648b8694cf4163eec54efa6c6d595212a25fc9555ace6dd2a8402907de5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 87316} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1dae0ca60fa8c222b03b56d875bece137690782bb37d4063d488e5200edad062', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 87337} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9209376, partition_space_available_1k=71881108, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2090872, cpu_share=0.28, mem_res=1583416, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=89416, cpu_share=0.03, mem_res=87172, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55028, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6801304, free=235804, buffers=276080, cached=8862448, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.0, five_min_avg=1.07, fifteen_min_avg=1.08, cpu_share=5.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4c4f5ec1973add3b72d13e9994e9094d4bcc4d5e03b14cf95f2440433dd12864', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 87926} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f24fde5b53bc41ad1f1bae9e5e1ff6a68dc681ad6568c86f22e13045d775ee2b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 87948} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9217432, partition_space_available_1k=71873052, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2096924, cpu_share=0.1, mem_res=1584804, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=89472, cpu_share=0.0, mem_res=87212, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55032, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6829740, free=214832, buffers=274692, cached=8856372, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.22, five_min_avg=1.56, fifteen_min_avg=1.24, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e4781456e94a8f77b73a4e8afb9ac05692c39b2fab4673ea22c6b4faedc4510e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 88505} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5b8aee6ac733e804170cf02331b2c944d79209583c0bc54809ae1126baf2cba2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 88527} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9230396, partition_space_available_1k=71860088, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2109788, cpu_share=0.09, mem_res=1581840, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=89564, cpu_share=0.0, mem_res=87260, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55028, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6815520, free=233272, buffers=273412, cached=8853432, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=1.41, fifteen_min_avg=1.21, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'eaca2a08e7c51eacb54e7fad1988c4256832b61e9100c5836e2623eb27efc3d1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 89074} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a3da8c5fa48dc9da7b383767ebc7f83131da7bbc67f96ec1224f6fc689734609', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 89095} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9234760, partition_space_available_1k=71855724, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2111540, cpu_share=0.06, mem_res=1581036, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=89544, cpu_share=0.0, mem_res=87260, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55032, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6805436, free=239064, buffers=273472, cached=8857664, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.9, five_min_avg=1.48, fifteen_min_avg=1.24, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f4b1c8664c46c4b07919cc31d30ad9b302a4060a3aef52690aacfba3afb81d25', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 89629} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '97f0c049c9df1dff89e853b312c6ea8d1ebdf45034d25cd382898c66d8784826', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 89650} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9239540, partition_space_available_1k=71850944, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2115524, cpu_share=0.08, mem_res=1581084, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=89544, cpu_share=0.0, mem_res=87260, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55040, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6811628, free=228044, buffers=273520, cached=8862444, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.54, five_min_avg=1.42, fifteen_min_avg=1.23, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e0980581413f5d4006cc88e06526293432118ec5dd839e6bc6c65168232e4d7a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 90184} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6d160e59d19c0ebfdb3e77005168550a53e79728f02726299c28cf069069d24d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 90207} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9244688, partition_space_available_1k=71845796, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2119580, cpu_share=0.06, mem_res=1581048, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71256, cpu_share=0.0, mem_res=68972, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55044, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6773508, free=261096, buffers=273576, cached=8867456, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.92, five_min_avg=1.25, fifteen_min_avg=1.18, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f6bc60c8f9b19ac15ac374478473058294af9700ad12e9a6f46f063333676cdd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 90808} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f98cba326e6531383f99b6f8db0cb24cdac8c09d47a7bf3c0402583fa7c586e2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 90830} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9253864, partition_space_available_1k=71836620, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2125408, cpu_share=0.1, mem_res=1581288, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71680, cpu_share=0.0, mem_res=69364, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55048, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6803564, free=224184, buffers=273520, cached=8874368, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.78, five_min_avg=1.16, fifteen_min_avg=1.16, cpu_share=5.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9c374f942b3c85cd6c03ee5d617d31d6e13063750012b4d96b4c62a72c505085', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 91364} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c2b02c20bf8e8f4c6b57039c542037ee2221ff6afcfbf14ec1459f127b966add', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 91385} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9259272, partition_space_available_1k=71831212, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2128780, cpu_share=0.06, mem_res=1581336, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71784, cpu_share=0.0, mem_res=69480, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55048, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6792564, free=243160, buffers=273304, cached=8866608, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.83, five_min_avg=1.13, fifteen_min_avg=1.15, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2ba8f8debaa44a6b24954364b02c58c69954031dfece902442f36192058c24ef', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 91916} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a060454672d800e94b6340e211a7e19dc3ff98211c52b5ecab943d4295d85d38', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 91938} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9265488, partition_space_available_1k=71824996, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2133780, cpu_share=0.06, mem_res=1581384, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71784, cpu_share=0.0, mem_res=69480, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55060, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6813664, free=215760, buffers=273340, cached=8872872, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.64, five_min_avg=1.02, fifteen_min_avg=1.1, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '19f15880a15aaefc81ec66f1cc8c5058e9357dfaa4c9ee4e5c7aee431a9b167d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 92492} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd8cf0d30660fc2bc93d4d8035a4c032f59ad66d7e183bfc7f0f3314c8cc34ffe', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 92513} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9268928, partition_space_available_1k=71821556, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2139024, cpu_share=0.08, mem_res=1581448, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71836, cpu_share=0.0, mem_res=69532, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55060, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6801228, free=234292, buffers=273036, cached=8867080, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.71, five_min_avg=0.96, fifteen_min_avg=1.08, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cf20f7457b55d3b0390762a8a649c07395117756e735b208b08bc66d7727f3e4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 93052} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a736b8ece0bfe9564772b93d60a0c229a5f24c5cc1ca7ccbd7f820b34fbe796b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 93073} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9273416, partition_space_available_1k=71817068, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2142564, cpu_share=0.06, mem_res=1581560, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71940, cpu_share=0.0, mem_res=69636, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55064, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6809156, free=224592, buffers=272996, cached=8868892, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.79, five_min_avg=0.93, fifteen_min_avg=1.06, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '42ce837b9284bc657f575b4dd5d2bca88f03a2d7dfdb154eb7891d67f1296844', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 93608} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '82f82e81873c9971b5ca8c347b41391e69246ca40904a676d79f2973fd4609ac', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 93630} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9278740, partition_space_available_1k=71811744, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2146756, cpu_share=0.06, mem_res=1581596, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=70952, cpu_share=0.0, mem_res=68648, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55068, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6803048, free=231104, buffers=272828, cached=8868656, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.59, five_min_avg=0.85, fifteen_min_avg=1.02, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9b06c21c145523ffed03ec41050402c003c59577df4e1a9a1569b2be2ea3d53f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 94161} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8297ab3ec674d9c94fd7fdd9cc5a8193a533edc06b4dbf80692d8f5c978dd53c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 94184} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9283384, partition_space_available_1k=71807100, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2149932, cpu_share=0.07, mem_res=1581740, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71068, cpu_share=0.0, mem_res=68760, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6792288, free=236980, buffers=272876, cached=8873492, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.63, five_min_avg=0.81, fifteen_min_avg=0.99, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7f3d7a7a87d0539f019c8d19e7876edb1ef8a585b8bd978ed4dd6af7a8d8df9e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 94757} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3be0d36f166ba742b81b59becc2bfccd8a878e330a842b1bd67358f73a3c3eac', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 94779} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9319812, partition_space_available_1k=71770672, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2184080, cpu_share=0.14, mem_res=1582736, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71196, cpu_share=0.0, mem_res=68888, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6786740, free=232016, buffers=272820, cached=8884060, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.82, five_min_avg=0.83, fifteen_min_avg=0.99, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0f50c7a28b501c3a6d768ec3858338a3cfe296779f6d416577f47cad0ceaacc6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 95311} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '95dca1688491343d6d1318294eeae1c4cd58a17e64e705a268da4c8dffab3c06', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 95332} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9324728, partition_space_available_1k=71765756, percentage_partition_space_used=11, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2186452, cpu_share=0.06, mem_res=1581976, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71328, cpu_share=0.0, mem_res=69020, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55076, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6800192, free=214068, buffers=272844, cached=8888532, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.84, five_min_avg=0.81, fifteen_min_avg=0.97, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '81fd186ff6893e2d7a5c68114689531bafe2866d3812fb6257cbcbf6bae2e1ed', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 95873} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '48e84e13429470153e776f33a33bd6b48356acfc0c8970401a936eef22750d15', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 95895} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9330488, partition_space_available_1k=71759996, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2189844, cpu_share=0.06, mem_res=1581968, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71380, cpu_share=0.0, mem_res=69072, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55080, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6790200, free=230732, buffers=272884, cached=8881820, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.7, five_min_avg=0.77, fifteen_min_avg=0.95, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '465943c2087e0dc39dcd606495ea67dec1917a054e6cf0bcf3ec2a145f43d72d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 96514} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '589860cb1e2f57f73998b5ff374c6e337d179b95eceb08fdde8f03c3ffc33c07', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 96535} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9336104, partition_space_available_1k=71754380, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2194040, cpu_share=0.09, mem_res=1582148, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71440, cpu_share=0.0, mem_res=69132, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6813396, free=234456, buffers=272924, cached=8854860, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.12, five_min_avg=1.1, fifteen_min_avg=1.05, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'af0e81bffca6c16048ffd3f1793c01073078b60ceadba9541ec723337c04c599', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 97067} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '23ba3de28d181dc6f2de67f4af412c1a09128fb5b8d53c6f14b0156cdbef728c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 97088} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9341504, partition_space_available_1k=71748980, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2198424, cpu_share=0.06, mem_res=1582256, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=70964, cpu_share=0.0, mem_res=68656, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55080, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6799224, free=243284, buffers=272964, cached=8860164, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.29, five_min_avg=1.05, fifteen_min_avg=1.03, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '00d5fe8263aa546a3dd860e047a73bd7b46bc012282ceec7d4d16cea9191dc1e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 97618} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'bcff68e19b1616a923ef0f103f4cc634aac451ed2721a1e8c2ccba19ba507fe9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 97639} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9346528, partition_space_available_1k=71743956, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2202060, cpu_share=0.06, mem_res=1582404, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71088, cpu_share=0.0, mem_res=68780, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55084, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6794500, free=242744, buffers=272984, cached=8865408, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.0, five_min_avg=1.01, fifteen_min_avg=1.02, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e44b125c0f46df19eb85b9013dacc8bcfc1aac631bf35760dfffff0f2ce8192b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 98176} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '36f2b5bbdab07eb2bc994f7f26dc9254f7b9919f73cb94d64e8c807234baf804', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 98197} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9346332, partition_space_available_1k=71744152, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2205164, cpu_share=0.06, mem_res=1582264, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71144, cpu_share=0.0, mem_res=68836, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55084, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6802296, free=235368, buffers=273036, cached=8864936, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.75, five_min_avg=0.93, fifteen_min_avg=0.99, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a274addb1a52b2b68246e6ac8e3a24035ea2498b29636ee3abd362123aff483e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 98740} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0ac4ea3cd450f7ddb3a549334d28ffe82907f918abcd24e7a632c2a04c661111', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 98762} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9350872, partition_space_available_1k=71739612, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2208800, cpu_share=0.06, mem_res=1582324, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71212, cpu_share=0.0, mem_res=68904, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55096, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6794408, free=238120, buffers=273072, cached=8870036, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.93, five_min_avg=0.96, fifteen_min_avg=1.0, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bc8ee1db5dcc0495abaffd0914187f3c3998c7c9ce2ac08b9f3bce42a05f4dbf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 99437} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'db10d3f95b4a636d16bc9b959554b33afbccff3acf86c4a8f352aa7758fa723a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 99459} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9362284, partition_space_available_1k=71728200, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2212828, cpu_share=0.11, mem_res=1582392, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71336, cpu_share=0.0, mem_res=69028, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55084, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6818608, free=230332, buffers=272724, cached=8853972, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.72, five_min_avg=1.15, fifteen_min_avg=1.06, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b90abf1d96cdea271cc37d234c524c2af38cc1e78abcf6bb0015fcfad71e0cbf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 100078} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '29a59d498936d440d112496eebf98bc1686c9554bc869a196c0a0d8842b23698', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 100099} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9376432, partition_space_available_1k=71714052, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2222760, cpu_share=0.14, mem_res=1583628, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71396, cpu_share=0.0, mem_res=69088, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55088, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6808012, free=244980, buffers=272516, cached=8850128, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.41, five_min_avg=1.57, fifteen_min_avg=1.2, cpu_share=7.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9f5bf661db94f927395ecd1e843e2c8cd5b8aa7b5373d349c27b4e818489ac7b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 100709} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c6b23441998a3bbaf46b82fbcdadddf30a11b643458cc7e3fe75f77745d98186', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 100732} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9385920, partition_space_available_1k=71704564, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2231488, cpu_share=0.08, mem_res=1587528, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=70960, cpu_share=0.0, mem_res=68652, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55092, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6840520, free=210288, buffers=272444, cached=8852384, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.81, five_min_avg=1.74, fifteen_min_avg=1.28, cpu_share=5.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4bc5cf72f2cf9ddccd6e2e8770226749663fa08e36d19ac0b4e0e07588a3cc6b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 101341} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '14b9aecb77ec237e4c1b06512ec777d010c614c062034e86088fcd1712bc64ab', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 101360} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9396132, partition_space_available_1k=71694352, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2235544, cpu_share=0.09, mem_res=1588140, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71092, cpu_share=0.0, mem_res=68784, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55092, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6832600, free=236816, buffers=272100, cached=8834120, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.36, five_min_avg=1.87, fifteen_min_avg=1.36, cpu_share=6.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 03:58:30] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6756 0.001391 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '39304686229d1352a2e879a988265e0b8874e9998aaf944e0bc5a00447d20890', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 101974} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1352114c03b166f42b839c3071d456ab52fb71c6083b8325441cb04150cf9f27', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 101996} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9402996, partition_space_available_1k=71687488, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2241580, cpu_share=0.07, mem_res=1590300, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71264, cpu_share=0.0, mem_res=68952, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55104, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6824716, free=238720, buffers=272144, cached=8840056, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.26, five_min_avg=1.65, fifteen_min_avg=1.32, cpu_share=4.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bb8e7834ae5388df32dfe7178e3c4f45cb59a9ba3a43c2481ea34dd235d395a3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 102545} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '842032ee02b86f9d85b106acfdaad2258e34d209e1efef6283ebd84f58d77e45', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 102566} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9416152, partition_space_available_1k=71674332, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2399172, cpu_share=0.12, mem_res=1748808, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71384, cpu_share=0.0, mem_res=69072, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55040, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6956916, free=247852, buffers=270976, cached=8699892, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.69, five_min_avg=1.42, fifteen_min_avg=1.26, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '20768015f1e01f231b6bfaeabe159a4a35ead5bac2ef866ff4661b31ea009595', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 103393} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '567cab823096a0850729f4e0dacc26e60fb3c562ee0fed74c2139939e7a036b3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 103415} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9421576, partition_space_available_1k=71668908, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2400260, cpu_share=0.06, mem_res=1747412, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71440, cpu_share=0.0, mem_res=69128, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55044, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6934132, free=264316, buffers=271576, cached=8705612, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.85, five_min_avg=1.33, fifteen_min_avg=1.24, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c9b3a7df96ebd8a8cb53c5320d0403340384ebf247323b9b5db631d661bd8ff6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 103978} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '45664c47f446c399b1d622d8f60e82bb8cb9318f660370cc2cd068bcb5283610', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 104001} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9428320, partition_space_available_1k=71662164, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2404316, cpu_share=0.07, mem_res=1747536, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71568, cpu_share=0.0, mem_res=69256, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55048, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6945136, free=246452, buffers=271620, cached=8712428, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.89, five_min_avg=1.27, fifteen_min_avg=1.22, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a1d2a37562242059653da8c4378aa05247dc23c1f930d292978ae307cf28e19c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 104529} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '03a487e8435b6e2628fd0ac9562e730b67675c050b60a7d4a8b1d45eb1e5cc20', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 104551} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9428260, partition_space_available_1k=71662224, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2408056, cpu_share=0.06, mem_res=1747592, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=70944, cpu_share=0.0, mem_res=68636, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55060, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6948328, free=242364, buffers=271664, cached=8713280, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.85, five_min_avg=1.18, fifteen_min_avg=1.19, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 04:04:14] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6756 0.001634 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '85f59d2d23885c65b333b6c3b4961368f351b73af45bfac4600eea4ae208f84a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 105650} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '847b09c3cdab13eb0fe76c16c8d36941234ef6931c60991b01a457c8fdb95008', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 105671} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9433376, partition_space_available_1k=71657108, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2410972, cpu_share=0.07, mem_res=1747712, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71120, cpu_share=0.0, mem_res=68804, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55056, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6930188, free=255208, buffers=271780, cached=8718460, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.2, five_min_avg=1.26, fifteen_min_avg=1.22, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1e2c08ad93ec4b3ce7c08516d70f43e0d31803895368da4a9cf40580e7395e36', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 106203} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b41a54e7a09ebe8e670fa03bb287b7bf278dd6abd66d68a811c52673ebedad44', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 106224} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9438444, partition_space_available_1k=71652040, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2414844, cpu_share=0.06, mem_res=1747864, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71248, cpu_share=0.0, mem_res=68932, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6939156, free=241264, buffers=271796, cached=8723420, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.91, five_min_avg=1.17, fifteen_min_avg=1.19, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6a8447ebb3b0071335fc398e610a7313f3ae84f5006d4fb71efb3b231fcb1b7f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 106764} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'aa56196d5c147999472ff9c7e8929ecfb1ccfc38819e6e6c15afaa109c8c66ce', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 106785} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9442508, partition_space_available_1k=71647976, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2417532, cpu_share=0.05, mem_res=1748324, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71308, cpu_share=0.0, mem_res=68992, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6942620, free=239744, buffers=271724, cached=8721548, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.85, five_min_avg=1.11, fifteen_min_avg=1.16, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f3a5b07ce2e5285f71552c842eb708d45735b7bbe643ae39c4f77e295f6fb4b1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 107359} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9128351670aa422f8b70d79a485b709b3ba88b0d759db9e2a1ae0accb73c8194', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 107380} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9477004, partition_space_available_1k=71613480, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2450392, cpu_share=0.13, mem_res=1750256, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71360, cpu_share=0.0, mem_res=69044, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6938964, free=233384, buffers=271684, cached=8731604, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.86, five_min_avg=1.07, fifteen_min_avg=1.15, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0b97e3110da6643f5f02842014e5fa4a8c851c9890c857fb4079b152bda861f0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 107940} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd2fa3c17df8f50231738bc1bb3e8551fba9454d987ba57699e47c29c1453c1ad', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 107972} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9483284, partition_space_available_1k=71607200, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2453136, cpu_share=0.07, mem_res=1750344, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71428, cpu_share=0.0, mem_res=69112, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55068, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6949860, free=230304, buffers=271728, cached=8723744, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.92, five_min_avg=1.04, fifteen_min_avg=1.13, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '3d662edbde1d5af0ffbb907eee42bea884ab12f253f7fcefd8c421711c12329b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 108559} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '14147d0d8a968be6f02412f42f052fdec74c1c368a2ad0191047f69377045b75', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 108581} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9489608, partition_space_available_1k=71600876, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2456044, cpu_share=0.09, mem_res=1749644, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=70944, cpu_share=0.0, mem_res=68632, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6936400, free=237364, buffers=271756, cached=8730116, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.83, five_min_avg=0.99, fifteen_min_avg=1.11, cpu_share=4.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'fa8b01163125dec391188d2d8499e6c18571102252c0b7d8e75a41d240ba62d0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 109117} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '77ce966077b9e5416735a003acf625240925c14caf9f4c191042cee4874744e3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 109137} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9488828, partition_space_available_1k=71601656, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2459180, cpu_share=0.06, mem_res=1749656, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71052, cpu_share=0.0, mem_res=68740, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55080, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6930636, free=243708, buffers=271820, cached=8729472, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.83, five_min_avg=0.96, fifteen_min_avg=1.09, cpu_share=3.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7ac7afd86622c356bc91bde996bb8b3e10ad021f907ac071a4fa855dfc923dae', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 109668} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9adf209dc376897672ca173a7e4e0891f71ee2f08dbd0d5f5017f9c825dec659', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 109689} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9493456, partition_space_available_1k=71597028, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2462808, cpu_share=0.06, mem_res=1749684, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71112, cpu_share=0.0, mem_res=68800, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55084, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6930304, free=239260, buffers=271836, cached=8734236, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.85, five_min_avg=0.97, fifteen_min_avg=1.08, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd514bf8771742b9de54c8275b7bde136b74f1c10eca133208d46c956cdafdfcd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 110221} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a1d4a9c82fad0ba05a94a3c5ff7e9b6fbf62dc34c8fa69a3e195b31e3e4e7f89', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 110242} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9498804, partition_space_available_1k=71591680, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2465472, cpu_share=0.06, mem_res=1749620, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71180, cpu_share=0.0, mem_res=68868, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55092, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6942428, free=228904, buffers=271872, cached=8732432, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.82, five_min_avg=0.93, fifteen_min_avg=1.06, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f005df0b74b497ca6568f73fbc24ec970959d98bd80c2ce6f9c4600617182b53', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 110772} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '15efa8dee51ebfc5a4dcb2e476f569536dc92cab84141f14f37180ed2510ed2f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 110793} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9502596, partition_space_available_1k=71587888, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2468200, cpu_share=0.05, mem_res=1749636, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71252, cpu_share=0.0, mem_res=68940, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55092, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6941200, free=226368, buffers=271908, cached=8736160, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.77, five_min_avg=0.91, fifteen_min_avg=1.04, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '84614289cc312e7ec703c009a4452d8cf26ea0d3452a3a819e74b960c507a7c3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 111386} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '398486695f4249eda8ce6bf16a9a8a93ec9cef66d7fd8f77a08a7d7bc704fe05', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 111410} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9509464, partition_space_available_1k=71581020, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2469616, cpu_share=0.08, mem_res=1749564, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71304, cpu_share=0.0, mem_res=68992, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55088, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6951052, free=237236, buffers=271932, cached=8715416, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.96, five_min_avg=0.95, fifteen_min_avg=1.05, cpu_share=5.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c25380e01b4b5681ad61d9e5fdfd477c01c98bbe92b1075b1842912d15b46334', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 111943} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8050bc4f3f3caf1bde9e24d3761f0da9902b637c535b3236f3f589797feb649c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 111964} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9514188, partition_space_available_1k=71576296, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2473200, cpu_share=0.06, mem_res=1749636, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=70944, cpu_share=0.0, mem_res=68632, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55092, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6938656, free=245564, buffers=271972, cached=8719444, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.72, five_min_avg=0.9, fifteen_min_avg=1.02, cpu_share=3.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1b49d7647549cd2a1c529c234f724b43b97713522e58da267a54feb8e4508871', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 112501} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9cf19beeaa38153420708167b86b25ce3a0d22ce3a0ae0ba1f6f1d0716e488d0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 112522} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9517300, partition_space_available_1k=71573184, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2475052, cpu_share=0.05, mem_res=1749564, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71012, cpu_share=0.0, mem_res=68700, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55100, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6935916, free=245348, buffers=272012, cached=8722360, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=0.64, five_min_avg=0.85, fifteen_min_avg=0.99, cpu_share=3.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8ac84dcea6c97c032599ddba5a0603651dc94258847ffec9e615e3926751ec20', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 113059} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cb3e2d16f92d1c46ea879797dace553a44b9fc7c1e2556ed22786e50219a3dd0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 113080} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9516408, partition_space_available_1k=71574076, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2477712, cpu_share=0.06, mem_res=1749564, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71080, cpu_share=0.0, mem_res=68768, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55100, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6932908, free=248888, buffers=272044, cached=8721796, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.05, five_min_avg=0.96, fifteen_min_avg=1.02, cpu_share=3.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b88bb6469754e8f2c7d462a9afa770f990347c5f50fcd38ca8a32e00e46711eb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 113802} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '880855e81e476150f73cd653f33f5e1cd3064b143001c472bb7b0dfa05ca68ff', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 113827} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9523316, partition_space_available_1k=71567168, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2481564, cpu_share=0.07, mem_res=1749648, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71144, cpu_share=0.0, mem_res=68832, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55100, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6952592, free=233256, buffers=271744, cached=8718044, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.65, five_min_avg=1.16, fifteen_min_avg=1.09, cpu_share=4.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e13e9493ceb28e3a15a1929c1c744c1934e743bd34b68b4dfdd8e4f3257dad57', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.41', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 114504} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '14dd112e67248b81798ac2132cc55294e0eccbb8f1b046acefb54561dcd62c6a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'b61bd0e21ed3c2a8799d2923d6ab82c26de85873ba3e6bc75060c786318305df', 'DetachKeys': '', 'Pid': 114526} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-3.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=9529308, partition_space_available_1k=71561176, percentage_partition_space_used=12, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2484564, cpu_share=0.06, mem_res=1749712, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=71204, cpu_share=0.0, mem_res=68892, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55108, cpu_share=0.01, mem_res=45132, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175636, used=6925388, free=248276, buffers=271880, cached=8730092, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.2, five_min_avg=1.13, fifteen_min_avg=1.08, cpu_share=4.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.41 - - [2025-08-28 04:20:14] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6755 0.001453