+++ date ++ echo 'INFO: =================== Thu Aug 28 02:08:15 AM UTC 2025 ===================' INFO: =================== Thu Aug 28 02:08:15 AM UTC 2025 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' +++++ ip route get 1 ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ cut -d / -f 1 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ ip addr show dev ens3 +++ head -n 1 +++ awk '{print $2}' +++ grep 'inet ' ++ DEFAULT_LOCAL_IP=10.0.0.19 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=256 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 256 ]] ++ [[ -z '' ]] +++ mount -t hugetlbfs +++ awk '/pagesize=2M/{print($3)}' +++ tail -n 1 ++ HUGE_PAGES_2MB_DIR= ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=openstack ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=rbac ++ AUTH_MODE=keystone ++ AUTH_PARAMS= ++ SSL_ENABLE=false ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_ALARM_ENABLE=True ++ ANALYTICS_SNMP_ENABLE=True ++ ANALYTICSDB_ENABLE=True ++ ANALYTICS_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICSDB_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_SNMP_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8081 ' +++ '[' -n '10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081 ' ']' +++ echo '10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081' ++ ANALYTICS_SERVERS='10.0.0.19:8081 10.0.0.38:8081 10.0.0.41:8081' +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9042 ' +++ '[' -n '10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042 ' ']' +++ echo '10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042' ++ ANALYTICSDB_CQL_SERVERS='10.0.0.19:9042 10.0.0.38:9042 10.0.0.41:9042' ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8086 ' +++ '[' -n '10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086 ' ']' +++ echo '10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086' ++ COLLECTOR_SERVERS='10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086' ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ CONFIGDB_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:8082 ' +++ '[' -n '10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082 ' ']' +++ echo '10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082' ++ CONFIG_SERVERS='10.0.0.19:8082 10.0.0.38:8082 10.0.0.41:8082' +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9161 ' +++ '[' -n '10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161 ' ']' +++ echo '10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161' ++ CONFIGDB_SERVERS='10.0.0.19:9161 10.0.0.38:9161 10.0.0.41:9161' +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9041 ' +++ '[' -n '10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041 ' ']' +++ echo '10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041' ++ CONFIGDB_CQL_SERVERS='10.0.0.19:9041 10.0.0.38:9041 10.0.0.41:9041' ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=false ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=false ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=false ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.20.0.25,10.20.0.17,10.20.0.254 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.20.0.25,10.20.0.17,10.20.0.254 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+=10.0.0.19:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+=10.0.0.38:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+=10.0.0.41:2181, +++ '[' -n 10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181, ']' +++ echo 10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181 ++ ZOOKEEPER_SERVERS=10.0.0.19:2181,10.0.0.38:2181,10.0.0.41:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:2181 ' +++ '[' -n '10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181 ' ']' +++ echo '10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181' ++ ZOOKEEPER_SERVERS_SPACE_DELIM='10.0.0.19:2181 10.0.0.38:2181 10.0.0.41:2181' ++ RABBITMQ_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+=10.0.0.19:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+=10.0.0.38:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+=10.0.0.41:5673, +++ '[' -n 10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673, ']' +++ echo 10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673 ++ RABBITMQ_SERVERS=10.0.0.19:5673,10.0.0.38:5673,10.0.0.41:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=false ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:6379 ' +++ '[' -n '10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379 ' ']' +++ echo '10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379' ++ REDIS_SERVERS='10.0.0.19:6379 10.0.0.38:6379 10.0.0.41:6379' ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=false ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.19 +++ local server_address=10.0.0.19 +++ extended_server_list+='10.0.0.19:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.38 +++ local server_address=10.0.0.38 +++ extended_server_list+='10.0.0.38:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.41 +++ local server_address=10.0.0.41 +++ extended_server_list+='10.0.0.41:9092 ' +++ '[' -n '10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092 ' ']' +++ echo '10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092' ++ KAFKA_SERVERS='10.0.0.19:9092 10.0.0.38:9092 10.0.0.41:9092' ++ KAFKA_SSL_ENABLE=false ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=10.0.0.19 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.19,10.0.0.38,10.0.0.41 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=false ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=false ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=false ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ /v3 == \/\v\2\.\0 ]] ++ [[ openstack == \o\p\e\n\s\t\a\c\k ]] ++ AUTH_MODE=keystone ++ [[ keystone == \k\e\y\s\t\o\n\e ]] ++ AUTH_PARAMS='--admin_password contrail123' ++ AUTH_PARAMS+=' --admin_tenant_name admin' ++ AUTH_PARAMS+=' --admin_user admin' ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ keystone == \k\e\y\s\t\o\n\e ]] + cat + [[ http == \h\t\t\p\s ]] + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + [[ database == \t\o\r\a\g\e\n\t ]] + export NODEMGR_TYPE=contrail-database + NODEMGR_TYPE=contrail-database + NODEMGR_NAME=contrail-database-nodemgr ++ echo DATABASE ++ tr - _ + ntype=DATABASE + [[ DATABASE == \V\R\O\U\T\E\R ]] + [[ DATABASE == \D\A\T\A\B\A\S\E ]] + htype=ANALYTICSDB + [[ DATABASE == \C\O\N\T\R\O\L ]] ++ get_listen_ip_for_node ANALYTICSDB +++ find_my_ip_and_order_for_node ANALYTICSDB +++ local server_typ=ANALYTICSDB_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.19,10.0.0.38,10.0.0.41 +++ local servers=10.0.0.19,10.0.0.38,10.0.0.41 +++ local server_list= +++ IFS=, +++ read -ra server_list +++ cut -d ' ' -f 1 ++++ get_local_ips ++++ cat /proc/net/fib_trie ++++ awk '/32 host/ { print f } {f=$2}' ++++ uniq ++++ grep -vi host ++++ sort ++++ tr '\n' , +++ local local_ips=,10.0.0.19,10.20.0.25,127.0.0.1,172.17.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.19'\''))' +++ local server_ip=10.0.0.19 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.19 ]] +++ [[ ,10.0.0.19,10.20.0.25,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.19, ]] +++ echo 10.0.0.19 1 +++ return ++ local ip=10.0.0.19 ++ [[ -z 10.0.0.19 ]] ++ echo 10.0.0.19 + hostip=10.0.0.19 + introspect_ip=0.0.0.0 + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + mkdir -p /etc/contrail + cat ++ format_boolean false ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ echo False + '[' -n '' ']' + cat + add_ini_params_from_env DATABASE_NODEMGR /etc/contrail/contrail-database-nodemgr.conf + local service_name=DATABASE_NODEMGR + local cfg_path=/etc/contrail/contrail-database-nodemgr.conf + local delim=__ ++ sed 's/^DATABASE_NODEMGR__//g' ++ cut -d = -f 1 ++ grep '^DATABASE_NODEMGR__.*__.*=.*$' ++ set -o posix ++ set ++ sort + local vars=DEFAULTS__minimum_diskGB + local section= + for var in $vars + local var_name=DATABASE_NODEMGR__DEFAULTS__minimum_diskGB + local val=2 ++ echo DEFAULTS__minimum_diskGB ++ sed s/DASH/-/g + local var_unsubst=DEFAULTS__minimum_diskGB ++ echo DEFAULTS__minimum_diskGB ++ sed 's/^\(.*\)__.*$/\1/' + local var_section=DEFAULTS + [[ '' != \D\E\F\A\U\L\T\S ]] + echo '[DEFAULTS]' + section=DEFAULTS ++ echo DEFAULTS__minimum_diskGB ++ sed 's/.*__\(.*\)$/\1/' + local var_param=minimum_diskGB + echo 'minimum_diskGB = 2' + cat /etc/contrail/contrail-database-nodemgr.conf [DEFAULTS] http_server_ip=0.0.0.0 log_file=/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log log_level=SYS_DEBUG log_local=1 hostip=10.0.0.19 db_port=9042 db_jmx_port=7200 db_use_ssl=False [COLLECTOR] server_list=10.0.0.19:8086 10.0.0.38:8086 10.0.0.41:8086 [SANDESH] introspect_ssl_enable=false sandesh_ssl_enable=false [DEFAULTS] minimum_diskGB = 2 + upgrade_old_logs contrail-database-nodemgr + local template=contrail-database-nodemgr ++ dirname /var/log/contrail/database-nodemgr + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/database-nodemgr + chmod 755 /var/log/contrail/database-nodemgr + mv -n '/var/log/contrail/*contrail-database-nodemgr.log*' /var/log/contrail/database-nodemgr/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/database-nodemgr + find /var/log/contrail/database-nodemgr -uid 0 -exec chown 1999:1999 '{}' + + exec /bin/sh -c '/usr/local/bin/contrail-nodemgr --nodetype=${NODEMGR_TYPE}' 08/28/2025 02:08:17.019 7f412a6a62c0 [contrail-database-nodemgr] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 08/28/2025 02:08:17.226 7f412a6a62c0 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 02:08:17.244 7f412a6a62c0 [contrail-database-nodemgr] [INFO]: SANDESH: INTROSPECT IS ON: 0.0.0.0:8103 08/28/2025 02:08:17.254 7f412a6a62c0 [contrail-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 08/28/2025 02:08:17.269 7f412a6a62c0 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 08/28/2025 02:08:17.269 7f412a6a62c0 [contrail-database-nodemgr] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/database-nodemgr/contrail-database-nodemgr.log] Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: CRITICAL:root:Exit data: {'ID': '709f771365bc32aa94aa04eea76e958ab23c620d102a558eaf5b7f5ae7d39006', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 83264} ERROR:contrail-database-nodemgr:Failed to connect to database by CQL and socket: Result: 1 INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-1.', _node_type='Database', _instance_id='0', _timestamp=1756346898903136, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Non-Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=[''], status='Down', description='none to Idle on EvStart')]), flag_infos=None, description='Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': '41cf8d4b600ff9b9f864bde7605a901482f452a2f5f9f30ea124d02243fb5118', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 83322} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': '13b36e2f8637b608dbd72244025dc8fd10bf7fbafc147a1f99876daaf4c68fd7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'repair', '-pr', 'ContrailAnalyticsCql'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 83458} ERROR:contrail-database-nodemgr:Failed to run cmd: 'nodetool -p 7200 repair -pr ContrailAnalyticsCql'. Error is: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:send_process_state_db: Sending UVE: NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-1.', _node_type='Database', _instance_id='0', _timestamp=1756346907259118, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=False, status=None, process_status=None, process_info=[ProcessInfo(process_name='cassandra', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-query-engine', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-database-nodemgr', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1756346894000000', last_stop_time='', last_exit_time='', core_file_list=[])], disk_usage_info=None, description=None, all_core_file_list=None, build_info='{"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-08-28 00:14:38.756636", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10583-1.novalocal", "build-id" : "v0+nightly", "build-number" : "nightly.el9"}]}', process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=13650280, partition_space_available_1k=67440204, percentage_partition_space_used=17, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=[], build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1256176, cpu_share=0.0, mem_res=1246860, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=7152, cpu_share=0.0, mem_res=5612, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54652, cpu_share=0.0, mem_res=45140, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=7134896, free=284536, buffers=601600, cached=8154608, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.99, five_min_avg=3.52, fifteen_min_avg=1.86, cpu_share=0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. deleted = False process_info = [ << process_name = cassandra process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-query-engine process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-database-nodemgr process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1756346894000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, ] build_info = {"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-08-28 00:14:38.756636", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-10583-1.novalocal", "build-id" : "v0+nightly", "build-number" : "nightly.el9"}]} >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. disk_usage_info = { /dev/vda1 : << partition_type = ext4 partition_space_used_1k = 13650280 partition_space_available_1k = 67440204 percentage_partition_space_used = 17 >> } all_core_file_list = [ ] process_mem_cpu_usage = { cassandra : << mem_virt = 1256176 cpu_share = 0.0 mem_res = 1246860 >>contrail-query-engine : << mem_virt = 7152 cpu_share = 0.0 mem_res = 5612 >>contrail-database-nodemgr : << mem_virt = 54652 cpu_share = 0.0 mem_res = 45140 >> } system_mem_usage = << total = 16175640 used = 7134896 free = 284536 buffers = 601600 cached = 8154608 node_type = database-node >> system_cpu_usage = << one_min_avg = 5.99 five_min_avg = 3.52 fifteen_min_avg = 1.86 cpu_share = 0 node_type = database-node >> >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.38:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.38:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.41:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.41:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.19:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1.:Database:contrail-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8103 start_time = 1756346897287865 collector_name = collector_ip = 10.0.0.19:8086 collector_list = [ 10.0.0.41:8086, 10.0.0.19:8086, 10.0.0.38:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-database-nodemgr:Session Event: TCP Connected INFO:contrail-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-ansible-os-3956-1. process_status = [ << module_id = contrail-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.38:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001016 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[ClientInit] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 0 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Client: Event[EvSandeshCtrlMessageRecv] => State[ClientInit] -> State[Established] INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:Extracted sandesh length: 0000001438 DEBUG:contrail-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[Established] DEBUG:contrail-database-nodemgr:Number of uve types in sandesh control message is 3 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:DatabaseUsage] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '547830a9c79aee38eb2de400fb901c68306cc23ecf773b47a0edc44fe3b29e63', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 91210} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open INFO:contrail-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-database-nodemgr', _source='cn-jenkins-deploy-platform-ansible-os-3956-1.', _node_type='Database', _instance_id='0', _timestamp=1756346959070792, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-database-nodemgr', instance_id='0', state='Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=['10.0.0.38:8086'], status='Up', description='ClientInit to Established on EvSandeshCtrlMessageRecv')]), flag_infos=None, description='')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo'), _seqnum=0) CRITICAL:root:Exit data: {'ID': 'e997f3ecca6625d3a5b063c0be06a19817250aac3d74190d6b8af252fefa1e52', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 91279} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=14877988, partition_space_available_1k=66212496, percentage_partition_space_used=18, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1361744, cpu_share=0.35, mem_res=1341544, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=11668, cpu_share=0.0, mem_res=9688, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=54036, cpu_share=0.02, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=8299796, free=310372, buffers=627052, cached=6938420, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.89, five_min_avg=3.93, fifteen_min_avg=2.11, cpu_share=17.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '466e9bdc1df3d74a9f18e51c290d0fa566f1d66c000d0c239c4504ee992e3d42', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 94939} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '99a906ec8fb04ae5571d3ba4bc47c74d72dac8c4ea9ae437e56aa7d8b81e988e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 94959} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=14855380, partition_space_available_1k=66235104, percentage_partition_space_used=18, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1362952, cpu_share=0.08, mem_res=1341520, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=11992, cpu_share=0.0, mem_res=10012, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53920, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=8557028, free=426008, buffers=626796, cached=6565808, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.17, five_min_avg=3.56, fifteen_min_avg=2.1, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'edb722b23d66dcbb76e5b7db5b172bc9d05f1e3d4aaa7e3203d41f2a808780cc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 97025} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd4d2ec8fa2ae215e29f2acbc3c42d3851f332d68233b8b017952f276054e8d18', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 97047} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=14861832, partition_space_available_1k=66228652, percentage_partition_space_used=18, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1370988, cpu_share=0.09, mem_res=1346776, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12112, cpu_share=0.0, mem_res=10132, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53912, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=8598736, free=295468, buffers=631640, cached=6649796, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.84, five_min_avg=3.09, fifteen_min_avg=2.02, cpu_share=7.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 02:11:24] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6757 0.001468 DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 02:11:46] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6757 0.001364 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '80d8c6080609cf3aa3582159a414906d6ecd7d9ab732abca27f905c5f59719ad', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 99768} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0ffcfafe0c2e1204c267645be05c204a1edfbf396f6d38e81f7b8b900969d2d8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 99790} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=15224796, partition_space_available_1k=65865688, percentage_partition_space_used=19, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1375164, cpu_share=0.07, mem_res=1348888, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12400, cpu_share=0.0, mem_res=10412, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53760, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=8550816, free=290332, buffers=638728, cached=6695764, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.59, five_min_avg=2.77, fifteen_min_avg=1.98, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 02:13:08] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6757 0.001491 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0d09f4b6b0cb029aa8f2f22177319846220725c560aee503baec95a1cea53486', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 102519} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '97a203a340e18eea6d338e3cf46d88961ed89ba63131549988ce4b3f8238d02f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 102541} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=16304636, partition_space_available_1k=64785848, percentage_partition_space_used=20, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1379284, cpu_share=0.13, mem_res=1351632, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=12520, cpu_share=0.0, mem_res=10532, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53576, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=8825616, free=403960, buffers=660748, cached=6285316, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.54, five_min_avg=2.85, fifteen_min_avg=2.06, cpu_share=12.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '32b7f596036764dc8accea2271db0971cbab624d66402fa9fbb4989942f67f3b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 104577} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd7efbd77b52d81ccad0c3097c32267fd3282365e2b3aaa0d199e100d8511fac2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 104600} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=16356520, partition_space_available_1k=64733964, percentage_partition_space_used=20, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1398928, cpu_share=0.15, mem_res=1362700, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13732, cpu_share=0.0, mem_res=11732, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=53412, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9002384, free=314020, buffers=660124, cached=6199112, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.94, five_min_avg=3.11, fifteen_min_avg=2.19, cpu_share=14.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c37d272f22af6529d68cf18ddc7cfc867b0c43a391670250c5430db67977fde0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 106503} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'fa4e879ce8333077aed29394f64d0b346055754e901e1f5433909ff62fe5b457', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 106525} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=18154436, partition_space_available_1k=62936048, percentage_partition_space_used=22, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1403768, cpu_share=0.07, mem_res=1365904, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13108, cpu_share=0.0, mem_res=11112, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=52396, cpu_share=0.02, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9041656, free=1364960, buffers=544148, cached=5224876, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.93, five_min_avg=2.99, fifteen_min_avg=2.21, cpu_share=11.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a25650a7b2b7bc34ccc6a3171b8fd706a2e62ef1558b97909bdcbed3d370f53a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 108548} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f85dc400edea0074f3f68ba0c85f7fb2f0e3bd427fb3898c58dbe339d9de5f57', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 108570} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=18170012, partition_space_available_1k=62920472, percentage_partition_space_used=22, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1410652, cpu_share=0.11, mem_res=1366104, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13216, cpu_share=0.0, mem_res=11220, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=52396, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9093024, free=1292496, buffers=544768, cached=5245352, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.19, five_min_avg=3.33, fifteen_min_avg=2.37, cpu_share=12.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e9336bd4031ca2fd35b709e62d3698da20dee590286edc5f6f1796c9d9c829ec', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 110679} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '50512d7ad3639479b500575312ae3921667732f60d2648b5056c0dd32f04bb00', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 110698} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=20072768, partition_space_available_1k=61017716, percentage_partition_space_used=25, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1413668, cpu_share=0.13, mem_res=1371196, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13220, cpu_share=0.0, mem_res=11224, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51936, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9096984, free=316320, buffers=461708, cached=6300628, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.9, five_min_avg=3.48, fifteen_min_avg=2.49, cpu_share=14.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5fd57a502bcb82d3fdbdbfb49ab49deca1fb099499904b38f106907ea954e686', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 112565} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '10059f2d5a4caf23a227f09ac53caf8dadf30d3b7f8d26e2d60d148683129c9f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 112587} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=19986520, partition_space_available_1k=61103964, percentage_partition_space_used=25, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1405720, cpu_share=0.06, mem_res=1373708, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13328, cpu_share=0.0, mem_res=11332, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51712, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9106716, free=1220012, buffers=374844, cached=5474068, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.86, five_min_avg=3.64, fifteen_min_avg=2.61, cpu_share=9.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2a7aef5306548cac4b3c763f516646df136c504c58443a99c04d1b55572d2aa6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 114311} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cf9e4eb8019e666eee41296e78519cd2a673184b5bc468cadc3ae56124323999', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 114333} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=19992680, partition_space_available_1k=61097804, percentage_partition_space_used=25, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1412576, cpu_share=0.06, mem_res=1377516, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13388, cpu_share=0.0, mem_res=11392, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51712, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9242836, free=1038496, buffers=377920, cached=5516388, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.49, five_min_avg=3.25, fifteen_min_avg=2.54, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c525dff7c0f87af457ee52eba791e16d3741362efeced58ca292554d97807066', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 116458} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '87ae4998a3ec77279146c66414669b6ff31add853e3d28ebe7cec1938cee0975', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 116480} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=20003364, partition_space_available_1k=61087120, percentage_partition_space_used=25, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1421992, cpu_share=0.08, mem_res=1382392, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13512, cpu_share=0.0, mem_res=11516, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51724, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9262044, free=1012340, buffers=378336, cached=5522920, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.87, five_min_avg=3.6, fifteen_min_avg=2.69, cpu_share=10.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4cf01c84b432581d3ab81f5bad671de71e28200ea46e62bfc7fed73c34818130', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 118501} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9b3f57adb1b6a6eb98b1571e725d79aaa64f4de7ca50f59ee142688ee7342c12', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 118522} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=20019124, partition_space_available_1k=61071360, percentage_partition_space_used=25, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1436400, cpu_share=0.1, mem_res=1389856, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13480, cpu_share=0.0, mem_res=11484, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51724, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9284376, free=959028, buffers=378916, cached=5553320, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.36, five_min_avg=3.35, fifteen_min_avg=2.66, cpu_share=11.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'fb3faf19b28842bde7c3bca2f45da14690514c0eb486f9f9ccaefc17274950e9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 120492} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '904f97ee41892f5753296f5dce59c77662c2089fc3dbced40a15c35ee95114f7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 120515} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=20035592, partition_space_available_1k=61054892, percentage_partition_space_used=25, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1449820, cpu_share=0.1, mem_res=1395824, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13536, cpu_share=0.0, mem_res=11540, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51728, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9315316, free=909588, buffers=379332, cached=5571404, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.21, five_min_avg=3.28, fifteen_min_avg=2.68, cpu_share=11.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '335d1cf06eee6ee25d7b0717fe670a010c9cd89eabd1cd63a653fad5f20d797a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 122315} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ff7ad4114ed6d6931af8f1f268d5d8bf4eedf3db526d8084f41d904e781e59ed', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 122375} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=20045060, partition_space_available_1k=61045424, percentage_partition_space_used=25, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1458604, cpu_share=0.07, mem_res=1400148, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13580, cpu_share=0.0, mem_res=11580, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51736, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9313932, free=901076, buffers=379568, cached=5581064, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.16, five_min_avg=2.94, fifteen_min_avg=2.6, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd3713f3f33d4a2fd6247f0860baa7b655e34e244a58b35d81d00b4d98b1858d0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 124162} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '869a5a23edeac5f3b9e8619474a58160b4e1f692902f024c300ce204468d15ee', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 124182} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=20882760, partition_space_available_1k=60207724, percentage_partition_space_used=26, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1460280, cpu_share=0.08, mem_res=1406080, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13624, cpu_share=0.0, mem_res=11628, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51492, cpu_share=0.02, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9348332, free=249636, buffers=246624, cached=6331048, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.45, five_min_avg=3.35, fifteen_min_avg=2.75, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '46a33a75ec2fc4570b39a8575a4c2dbd77da3fc136a103db1e8c40162b12d5fc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 126107} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '01e45606c103cf82b38856de3debf80b07f3e9791457118f4128ea3964fd2612', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 126130} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21854440, partition_space_available_1k=59236044, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1439652, cpu_share=0.05, mem_res=1407816, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13676, cpu_share=0.0, mem_res=11688, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51208, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9313060, free=1523528, buffers=127348, cached=5211704, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.44, five_min_avg=3.23, fifteen_min_avg=2.74, cpu_share=11.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ef9c011aebe4c982704d7357b6da718fe8760cee40ec62f50d09bdfd39f1e2b9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 128006} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '16ba79bd9cfe130cc010c7edfb6da51e98f3185b1b5322a4f1cf72d7911ca9f6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 128065} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21860056, partition_space_available_1k=59230428, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1445972, cpu_share=0.06, mem_res=1411188, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13732, cpu_share=0.0, mem_res=11744, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51216, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9332040, free=1493908, buffers=127564, cached=5222128, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.56, five_min_avg=3.01, fifteen_min_avg=2.7, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7970d8d7d363be0e96106a0e09dba70b420538bf326d78b97089f425747990e7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 130017} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '44508343ce3c77eabd878f4cc060c47ba6e52815080f8ae11302309143f362d6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 130046} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21871688, partition_space_available_1k=59218796, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1454564, cpu_share=0.07, mem_res=1415924, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13708, cpu_share=0.0, mem_res=11720, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51220, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9363464, free=1450000, buffers=127992, cached=5234184, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.03, five_min_avg=2.75, fifteen_min_avg=2.62, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6471df9244c5a9a6c55be985f77f006535662f5b17d7ba85c4874630330434ff', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 131899} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'abdc00eb773ee9e41952b9c552a67f0f292daa34226cb12290a49709439e7912', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 131960} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21880400, partition_space_available_1k=59210084, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1462732, cpu_share=0.06, mem_res=1419660, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13728, cpu_share=0.0, mem_res=11740, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51220, cpu_share=0.02, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9351544, free=1452856, buffers=128384, cached=5242856, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.91, five_min_avg=2.6, fifteen_min_avg=2.58, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '95d409e776e6fd10d6ef5503914b602d75459d1cc5270bbb2fd757ac804e0ade', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 133709} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'abccf89fc4e66f44451d3eeae4aad12c00535fab4b73d8040a5f678830392a43', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 133756} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21890332, partition_space_available_1k=59200152, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1472956, cpu_share=0.07, mem_res=1425116, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13744, cpu_share=0.0, mem_res=11756, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51228, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9382164, free=1407312, buffers=128940, cached=5257224, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.94, five_min_avg=2.47, fifteen_min_avg=2.54, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'beb8cefd13603def4a239b2a27a86ac0a765709ac13b381c9cc8926b8fb1ae13', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 135666} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2a35e9426a7d4db136996c3a5011dc142a6f569ed1d204c3e90fb91494258ad7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 135705} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21896476, partition_space_available_1k=59194008, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1478728, cpu_share=0.06, mem_res=1427912, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13828, cpu_share=0.0, mem_res=11840, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51232, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9370952, free=1414764, buffers=129208, cached=5260716, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.65, five_min_avg=2.28, fifteen_min_avg=2.46, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '49b34627b70effd2b1517b9afc9c72c4b35fd61e74deaf7faf5e69c77b2f373e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 137748} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '331334f67b0d0f7e24f49a9201e837b0afe9d056bff5a629f1d099415645e5eb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 137792} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21931380, partition_space_available_1k=59159104, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1512644, cpu_share=0.17, mem_res=1429664, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13852, cpu_share=0.0, mem_res=11864, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51236, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9391764, free=1351848, buffers=130104, cached=5301924, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.76, five_min_avg=2.89, fifteen_min_avg=2.65, cpu_share=12.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f4e4c0fe1694a899960b737d52f4aaf499d10ab9c5c4afd2bd39f26f45124c5e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 139714} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '51d9848ece03133205f4334e9dea077aec1b1f32e497749ceb42311e8731fa06', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 139737} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21944940, partition_space_available_1k=59145544, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1514260, cpu_share=0.06, mem_res=1428476, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13856, cpu_share=0.0, mem_res=11868, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51244, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9385516, free=1344336, buffers=130312, cached=5315476, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.82, five_min_avg=2.66, fifteen_min_avg=2.59, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'de496bc013fd2cbf1b8a50fac555bb21c5693b8980478a72688294b2ede34b33', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 141530} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '19c7c60f4355d078bbbbd74308cc132489d47758e2945661ca0b94180ddbbd4f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 141552} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21943564, partition_space_available_1k=59146920, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1518664, cpu_share=0.06, mem_res=1428524, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13804, cpu_share=0.0, mem_res=11816, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51248, cpu_share=0.02, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9369296, free=1351644, buffers=130596, cached=5324104, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.0, five_min_avg=2.42, fifteen_min_avg=2.51, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a93a8e6dfb93a982d28deb771a1b6a077b20c4c58d06e120782f4addec290553', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 143389} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9f972d6a73737f16a0f0a5017060df1d747124a71098ca5a5b9661e95d1417a1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 143411} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21957276, partition_space_available_1k=59133208, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1524908, cpu_share=0.09, mem_res=1429680, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13840, cpu_share=0.0, mem_res=11852, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51252, cpu_share=0.01, mem_res=45136, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9404404, free=1300176, buffers=131288, cached=5339772, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.24, five_min_avg=2.41, fifteen_min_avg=2.5, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '67dadba1669e4ed3989755a064ed479cd8a92079772759f72289e7e1979f9f63', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 145336} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2668ef4006e06a9168c396ba11b1cea6fa51bb6f8718a788206a7f1b5e16d57e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 145358} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21966460, partition_space_available_1k=59124024, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1531052, cpu_share=0.06, mem_res=1432004, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13848, cpu_share=0.0, mem_res=11860, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51348, cpu_share=0.01, mem_res=45228, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9414096, free=1276648, buffers=131508, cached=5353388, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.5, five_min_avg=2.17, fifteen_min_avg=2.4, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '857ef363687e64972dbb3000e9fea57f8cde8b300ccad21cac5c61ee501884c0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 147334} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f51ab41aa59444e7647bb587ba821fdef34b3dc1794850c5ded3e403fe39dca9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 147355} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21977136, partition_space_available_1k=59113348, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1536352, cpu_share=0.07, mem_res=1432960, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13852, cpu_share=0.0, mem_res=11864, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51392, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9423568, free=1254188, buffers=131888, cached=5365996, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.9, five_min_avg=2.14, fifteen_min_avg=2.38, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c9837635fa13348e4035583ca5f943993b9430ee79e5556af84e73c85dace82e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 149323} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e942b4701456ec587770a1c0b8d9e5df9cb0de56b0718cc0cabf79c89b573462', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 149345} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21981824, partition_space_available_1k=59108660, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1543700, cpu_share=0.06, mem_res=1435736, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13860, cpu_share=0.0, mem_res=11872, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51392, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9410824, free=1257864, buffers=132188, cached=5374764, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.11, five_min_avg=2.19, fifteen_min_avg=2.38, cpu_share=9.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b9e7bbb791c5ad8b4b0cc2cc5a2ca5ade319b653ee6c51227c3cb48bb19862eb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 151225} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '91bee7a0e19159b50b6b5e478520da3d4540cef336d6eab8d0c3507b0c429657', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 151247} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=21994048, partition_space_available_1k=59096436, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1551248, cpu_share=0.07, mem_res=1438196, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13888, cpu_share=0.0, mem_res=11900, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51400, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9422940, free=1232712, buffers=132776, cached=5387212, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.03, five_min_avg=2.12, fifteen_min_avg=2.34, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '032267f7328276bac681521dd9746386e3c6b416433deb2505692b2c0d6af72a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 153001} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ea67229fe8742504e68c2ee786665e7745ac41a45ad34e8754ecdc6be17a2971', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 153023} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22012356, partition_space_available_1k=59078128, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1563284, cpu_share=0.07, mem_res=1434032, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13824, cpu_share=0.0, mem_res=11836, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51408, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9456440, free=1173264, buffers=133160, cached=5412776, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.74, five_min_avg=2.03, fifteen_min_avg=2.3, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a2baf03aac592cd908314a72cd1e87fa1a91f9f6fabe39877f688ea7c96d53d6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 154984} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd350ff76ae40af8303a09470ae906d4e71071648abd28bc44b5c2933577155bd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 155008} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22017820, partition_space_available_1k=59072664, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1566972, cpu_share=0.07, mem_res=1433108, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13828, cpu_share=0.0, mem_res=11840, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51412, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9431364, free=1195368, buffers=133568, cached=5415340, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.67, five_min_avg=1.92, fifteen_min_avg=2.24, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9d67af2682b101f08b3964aae80737c603252184bce917482f948ab9d0e68c5c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 156969} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '90612a0d0ab5839ff982d174e9553bb201bee77e744364604cdc6a7ea07f6ad0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 156991} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22029324, partition_space_available_1k=59061160, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1572120, cpu_share=0.08, mem_res=1433336, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13848, cpu_share=0.0, mem_res=11860, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51416, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9432868, free=1181272, buffers=134176, cached=5427324, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.46, five_min_avg=1.83, fifteen_min_avg=2.19, cpu_share=9.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9f637329d27ec025f596050f54fcebcd5e738fe59c58bcf770eee18b9ab5acf6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 158927} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e58f07403ade7c792cef03c30f00408f639c93a3ae2c1f6eb163f7501bf4c252', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 158948} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22031120, partition_space_available_1k=59059364, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1575448, cpu_share=0.05, mem_res=1433364, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13916, cpu_share=0.0, mem_res=11928, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51424, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9440244, free=1172456, buffers=134388, cached=5428552, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=1.8, fifteen_min_avg=2.16, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '37c124f4d4d58c8f60438917636994209f1fdaff4f8629be8a29a6167e719524', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 160692} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '4ad3290604257c1e622fab7d6a18012d2477d741a9a6090897aac197fca710ab', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 160718} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22037588, partition_space_available_1k=59052896, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1578848, cpu_share=0.05, mem_res=1433308, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13932, cpu_share=0.0, mem_res=11940, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51424, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9427996, free=1175976, buffers=134624, cached=5437044, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.53, five_min_avg=1.72, fifteen_min_avg=2.1, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9da0d29b4117babea1baf62219b17cad1625674a28760f2776b6092383015fb5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 162597} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '11b1a24ac79465c5104a1875b4c630231e0ddbaf36b3ba80569e52db1b8b9d55', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 162619} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22048048, partition_space_available_1k=59042436, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1585740, cpu_share=0.1, mem_res=1433560, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13948, cpu_share=0.0, mem_res=11956, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51432, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9456272, free=1135624, buffers=135336, cached=5448408, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.26, five_min_avg=2.1, fifteen_min_avg=2.2, cpu_share=11.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6deda659a6e2ae930457bf7ec59d3b11f500117cf84760265c05cb7832879c85', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 164555} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '63372a775c353e3778a2ddeb2a8c29805d3caf3983c09b3212fb05a63ecb2009', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 164576} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22054516, partition_space_available_1k=59035968, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1589148, cpu_share=0.06, mem_res=1433584, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13920, cpu_share=0.0, mem_res=11928, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51440, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9457980, free=1127124, buffers=135496, cached=5455040, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.33, five_min_avg=2.04, fifteen_min_avg=2.17, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cf4a3d62da84d258149d747d268d9e082e255b5393ca768cc29c8a5c5190b904', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 166508} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e19458dc5c0ec99dfdc711317154e5f3ce293c1ecc98fde62f4e11895d9f6297', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 166532} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22051028, partition_space_available_1k=59039456, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1592324, cpu_share=0.06, mem_res=1433480, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13936, cpu_share=0.0, mem_res=11944, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51444, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9457032, free=1130692, buffers=135648, cached=5452268, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.58, five_min_avg=2.16, fifteen_min_avg=2.21, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'fd296a718fe7a8a96655d1587c60a65c9e039c9462ed994bfb164509d3ba5e67', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 168498} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '98305328c623cff591f706ab6e288199c2246abeae5cbbe8a6b9c4974d27fee9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 168524} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22061872, partition_space_available_1k=59028612, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1597036, cpu_share=0.07, mem_res=1433692, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13956, cpu_share=0.0, mem_res=11964, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51452, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9481096, free=1094496, buffers=136112, cached=5463936, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.5, five_min_avg=2.39, fifteen_min_avg=2.27, cpu_share=9.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b0e4abb9d8ee759e332fccb37f04707720343087e1e3a4b6c5151752d68f646a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 170223} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6e5e31b124b8992d32fecea255975a9c7fa76fdbc807b43d148dc15a465a7522', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 170246} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22067136, partition_space_available_1k=59023348, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1602568, cpu_share=0.07, mem_res=1433808, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13968, cpu_share=0.0, mem_res=11976, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51444, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9467024, free=1103360, buffers=136564, cached=5468692, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.54, five_min_avg=2.3, fifteen_min_avg=2.25, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '72b33f124fc371fbf97b2f47aa82fcd7af91982dbc2a05377e21412c168f9324', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 172169} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '35f4b6e6a47030199208af3a1c64bce2df4fa6d573dcc915131c5a50fbf221c1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 172189} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22079364, partition_space_available_1k=59011120, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1607992, cpu_share=0.08, mem_res=1433728, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13968, cpu_share=0.0, mem_res=11976, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51456, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9493276, free=1063604, buffers=137084, cached=5481676, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.49, five_min_avg=2.32, fifteen_min_avg=2.26, cpu_share=9.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '79904a0b02193bc7186d5c7c8f948f4219aebb1a64ae83b826617f1e815a5a90', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 174178} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a9a94047088c1386a2b2ab89e3887280b0db2c836975ad4305ccf76707b15c30', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 174199} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22092428, partition_space_available_1k=58998056, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1614204, cpu_share=0.08, mem_res=1433808, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14000, cpu_share=0.0, mem_res=12008, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51460, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9502552, free=1043180, buffers=137852, cached=5492056, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.53, five_min_avg=2.36, fifteen_min_avg=2.28, cpu_share=10.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '3bb96e148d6ead61e665247be0fbe967802b432638f556499f2b2773228160eb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 176169} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ba76793204401576e7e1370b6bde85a7f6577897d9e9332a78d390216d8668ee', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 176191} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22096312, partition_space_available_1k=58994172, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1618960, cpu_share=0.07, mem_res=1434676, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13976, cpu_share=0.0, mem_res=11984, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51464, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9493356, free=1043020, buffers=138212, cached=5501052, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.75, five_min_avg=2.18, fifteen_min_avg=2.22, cpu_share=9.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b31c9960ec3ea3ebc8d641ca4d1694d89284a2e0ba8f85e0cb009981c9086f11', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 178154} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '094bd7a19ebbc56396eac9a18b484a060723f22fe8abda8fbffa4d07cae094c1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 178178} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22119884, partition_space_available_1k=58970600, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1658812, cpu_share=0.12, mem_res=1440392, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13984, cpu_share=0.0, mem_res=11992, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51468, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9500012, free=998800, buffers=138508, cached=5538320, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.76, five_min_avg=2.1, fifteen_min_avg=2.19, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'da155c7aaac6b14b434774b314767ac643d99dc2f5e8f3ff11e97eae8e968c51', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 179840} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '984649cfe59c5366df99f6ecb44205083d401281ebdf1461347c6bb9699f7c8a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 179881} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22140076, partition_space_available_1k=58950408, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1662568, cpu_share=0.08, mem_res=1439600, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13984, cpu_share=0.0, mem_res=11992, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51468, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9510060, free=981256, buffers=138980, cached=5545344, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.77, five_min_avg=2.01, fifteen_min_avg=2.15, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '600a216ea1ab16d5e51a33e672ec484762eeef5e7326174772bfca9aa002c178', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 181760} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3d8553e576ebd7369100e2d5455d5c8f5c2e1b79518a0fa0fa17544b254f6ae0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 181805} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22148332, partition_space_available_1k=58942152, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1666824, cpu_share=0.06, mem_res=1439600, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14004, cpu_share=0.0, mem_res=12012, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51480, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9525304, free=957428, buffers=139144, cached=5553764, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.29, five_min_avg=1.84, fifteen_min_avg=2.08, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '01f4db8cf9306fcc5a20fdd57a50a6be21c89ac872fffb7a602205f61048148e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 183718} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3b7f2f1cf3594cbae1f06d6387a3a26f57dfdb07d2bdee2dfa55a9cf99f5bdbd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 183753} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22149660, partition_space_available_1k=58940824, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1670232, cpu_share=0.05, mem_res=1439528, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14008, cpu_share=0.0, mem_res=12016, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51484, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9521028, free=960312, buffers=139304, cached=5554996, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.93, five_min_avg=1.89, fifteen_min_avg=2.08, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '74dd23ad590d0dfae460a48ce4050c40f8f6c3089b6442c4b9a79bd65c609acf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 185713} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd08718c1d6d250926d43af6e1d1fe2f9323471f7b04d52158f23dbbe6515b93a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 185753} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22160036, partition_space_available_1k=58930448, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1674304, cpu_share=0.07, mem_res=1439504, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14020, cpu_share=0.0, mem_res=12028, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51488, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9545628, free=922644, buffers=139668, cached=5567700, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.27, five_min_avg=1.95, fifteen_min_avg=2.09, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f7f92df8905e6b9b5da2cfe3dd7259b2400ae4af43db4dae3fc3143b7332ecb5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 187681} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd2d8beb5133db896db04e75e4d516717c6797a29357c541cb8f42f1f02962af1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 187703} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22168992, partition_space_available_1k=58921492, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1679724, cpu_share=0.07, mem_res=1439656, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=13984, cpu_share=0.0, mem_res=11992, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51492, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9530560, free=929180, buffers=140040, cached=5575860, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.45, five_min_avg=1.77, fifteen_min_avg=2.02, cpu_share=9.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd52e8f02c29b3835409ddd38442909b398b204d15f57d0f740e859eb405462e3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 189442} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '59fadcb4394b301f3b57c6e58b3a99cc186e23056fefa1a2a70a952f6538d45f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 189499} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22178936, partition_space_available_1k=58911548, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1683148, cpu_share=0.06, mem_res=1439660, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14004, cpu_share=0.0, mem_res=12012, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51496, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9529488, free=920612, buffers=140228, cached=5585312, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.44, five_min_avg=1.71, fifteen_min_avg=1.98, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '229a5ec56e9358d425257e5e63447b4b71ad809528dfa12d7ed058e5de55d15d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 191306} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'f6e8aae796a16a7a2653790d80750bd5dda3cb1ff0ee1ac0e4e39c7dbd473796', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 191328} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22183276, partition_space_available_1k=58907208, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1686940, cpu_share=0.06, mem_res=1439668, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14020, cpu_share=0.0, mem_res=12028, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51504, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9560092, free=870704, buffers=145904, cached=5598940, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.26, five_min_avg=1.61, fifteen_min_avg=1.93, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '10ba1abacc9cc5eea53b13c712ef65c85d6aff9e0e6049040992cbfc6787be4a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 193268} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8b72290904778965cb7a3871e6972625aff541327888ab8cd6b3b47f0e36aef1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 193291} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22195144, partition_space_available_1k=58895340, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1692840, cpu_share=0.07, mem_res=1440160, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14036, cpu_share=0.0, mem_res=12044, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51508, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9552108, free=871840, buffers=146272, cached=5605420, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.72, five_min_avg=1.68, fifteen_min_avg=1.93, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '02ef56a3b9d8a58508acd4fa16738673cd94bb3384984de6a9e140bb5de67698', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 195243} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c22fc92b0d5e08b3b14d644f1df1ec23b9f84fe9939252c747ba38d8761aaf20', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 195264} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22197940, partition_space_available_1k=58892544, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1696120, cpu_share=0.05, mem_res=1440576, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14064, cpu_share=0.0, mem_res=12072, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51512, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9550812, free=869328, buffers=146444, cached=5609056, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.72, five_min_avg=1.89, fifteen_min_avg=1.98, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cfe053993ef5bf20a0293119028d0cbcff2ffc534e0be9770eaf4fe61de3781c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 197193} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '23c16a71f3bc99c5cc45a998380ac68de380c2e2188a38e1a2136e594a3e61de', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 197215} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22205436, partition_space_available_1k=58885048, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1702432, cpu_share=0.06, mem_res=1442528, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14064, cpu_share=0.0, mem_res=12072, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51516, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9550872, free=860092, buffers=146604, cached=5618072, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.98, five_min_avg=1.82, fifteen_min_avg=1.95, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b9a31bf134bb999fe0765a929c85925f59fdba3b6e1e9c7b0bbfd0ac65b2aa7f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 199024} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '09ffdc6bb106c4c3411a4cebde257845267eb5c0bea4d63b72e0f42095eb527e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 199048} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22216956, partition_space_available_1k=58873528, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1707952, cpu_share=0.06, mem_res=1443732, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14020, cpu_share=0.0, mem_res=12024, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51524, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9544824, free=855516, buffers=146828, cached=5628472, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.52, five_min_avg=2.15, fifteen_min_avg=2.04, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0c247dad95bbcc8752e2488e4972f837c4d236d8892dc5e045a77b9db11a59c8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 200952} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5dd5562a3688d8679bed185390acdf8ef1bcec7f3896ef1f3c6caa8b64bd5bd0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 200974} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22244880, partition_space_available_1k=58845604, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1734588, cpu_share=0.14, mem_res=1449580, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14024, cpu_share=0.0, mem_res=12028, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51524, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9587300, free=781632, buffers=147828, cached=5658880, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.07, five_min_avg=2.49, fifteen_min_avg=2.16, cpu_share=13.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1b38c278e9b4231754a7c29d0c1264814921b230ac28f82b6832cbd7b7fc5bc0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 202990} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8de5d438c3b0ca1c42f47bac025a1bd884eebf9ca70c1beabf39c8f9c339a97a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 203013} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22266420, partition_space_available_1k=58824064, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1741120, cpu_share=0.12, mem_res=1448700, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14028, cpu_share=0.0, mem_res=12032, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51532, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9597892, free=744900, buffers=148900, cached=5683948, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.18, five_min_avg=2.5, fifteen_min_avg=2.19, cpu_share=14.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c533518e5fe92500ae98ac22d9952fcf15780f6e6285e7d6410f8ed00a982b49', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 204966} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3d46ffdf613b12370470a7fb9e637c5ef74403c79016905babe6138af748c144', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 204988} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22269916, partition_space_available_1k=58820568, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1745512, cpu_share=0.06, mem_res=1448740, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14040, cpu_share=0.0, mem_res=12044, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51536, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9578540, free=760672, buffers=149540, cached=5686888, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.65, five_min_avg=2.43, fifteen_min_avg=2.18, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bb4ea101818b6f966b8210e4327b6144ff128f83a3deade9432734c072e0a134', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 206824} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '05869118e167136cd426c608080a4cf05f1da827955118fabd11d4f7dccc930c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 206885} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22279248, partition_space_available_1k=58811236, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1750260, cpu_share=0.06, mem_res=1448664, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14048, cpu_share=0.0, mem_res=12052, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51540, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9581104, free=749064, buffers=149688, cached=5695784, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.72, five_min_avg=2.46, fifteen_min_avg=2.2, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd32dc3c8624708309f6df8eafb7f8acba9748c1ed10d32b2690b1c3987841604', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 208621} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'eaa4aeeb076132ede2b9116fd6a1b02955ad59a8f58bfca29af860c4b1f86bd1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 208642} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22285292, partition_space_available_1k=58805192, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1757064, cpu_share=0.08, mem_res=1449592, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14064, cpu_share=0.0, mem_res=12068, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51544, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9604176, free=711348, buffers=150548, cached=5709568, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.47, five_min_avg=2.41, fifteen_min_avg=2.2, cpu_share=9.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7d6640ac0ad3357bab70171a4894a95a7a97cd33eef8f0f94623e3745bed63b7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 210579} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '51e20f3f476a0a1e476fb340b9b19e0e96891baf9f6ae3e7f7e6e8bd8fd856e1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 210600} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22292512, partition_space_available_1k=58797972, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1760968, cpu_share=0.06, mem_res=1449344, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14040, cpu_share=0.0, mem_res=12044, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51548, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9592540, free=715264, buffers=150736, cached=5717100, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.49, five_min_avg=2.4, fifteen_min_avg=2.21, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dc85339edd6a400b59a42b9260efa4cd872a3058b0e7db4bf1237945fb2da478', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 212536} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c0c285fe5b752bfc899cc2f4c9901f87260d336b0caecd7b457226e150c2ea7a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 212557} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=22294284, partition_space_available_1k=58796200, percentage_partition_space_used=27, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1763636, cpu_share=0.05, mem_res=1449076, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14064, cpu_share=0.0, mem_res=12068, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51556, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9606152, free=700424, buffers=150876, cached=5718188, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.03, five_min_avg=2.27, fifteen_min_avg=2.17, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4b603f98134fabf138e44fcf8b2e9e6b8b3c2969670cf7440adef5a5d0277ebd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 214549} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1c40aba01614e222e978d86410e1fc31ad401066df78f5952c26b4234615872a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 214570} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23230228, partition_space_available_1k=57860256, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1605820, cpu_share=0.07, mem_res=1449080, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14064, cpu_share=0.0, mem_res=12072, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51416, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9605852, free=810072, buffers=110236, cached=5649480, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.17, five_min_avg=2.46, fifteen_min_avg=2.24, cpu_share=11.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '159555e9fd63177bf69c2e15bb2372d634bb079c008f45a039830a6c069e26db', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 216212} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a8dceb0c7f6716fd40a23ab89aae34ae729211ed4eecb5af38bdc95d54f00d73', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 216235} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23232372, partition_space_available_1k=57858112, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1609872, cpu_share=0.06, mem_res=1448992, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14084, cpu_share=0.0, mem_res=12092, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51424, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9588104, free=820768, buffers=110380, cached=5656388, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.93, five_min_avg=2.21, fifteen_min_avg=2.16, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0c41614e8383a06f6e3a372ea328bcb158f6a2c03b43f28f01a633a3019187d3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 218161} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '85ad7761b1cb36cd504a0b68dbb08271724bc19457012b91009e93ef08c0ba03', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 218183} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23238580, partition_space_available_1k=57851904, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1613372, cpu_share=0.05, mem_res=1449084, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14092, cpu_share=0.0, mem_res=12100, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51428, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9602784, free=798960, buffers=110576, cached=5663320, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.51, five_min_avg=2.05, fifteen_min_avg=2.11, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd7a1bce092ec7da83be17dc9ab06465da863ef690315809cdb18b2b510362864', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 220182} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6ad7ebba2675d6c34b45dcc77f5c453322c760ba73c6d97c9fb6ef564cb139cb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 220203} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23267032, partition_space_available_1k=57823452, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1643468, cpu_share=0.13, mem_res=1444948, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14108, cpu_share=0.0, mem_res=12116, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51432, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9616092, free=745588, buffers=111044, cached=5702916, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.13, five_min_avg=2.38, fifteen_min_avg=2.22, cpu_share=10.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e7306bfec251664c1f38c13411964ce9cdf8f82c7e1762916482cd222a532539', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 222179} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ee288cb6e71c77c12213739b174fdbc34cbae679b6356debd9f5a391676aeda9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 222200} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23278288, partition_space_available_1k=57812196, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1648368, cpu_share=0.07, mem_res=1444240, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14052, cpu_share=0.0, mem_res=12060, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51436, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9610732, free=737660, buffers=111524, cached=5715724, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.75, five_min_avg=2.4, fifteen_min_avg=2.23, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5a6dcbc42ef9a471ab6254760bdc6f82e7addc7fa86f59caecb8bb2a2af8c9f3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 224156} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '68d9ce05723829f90db4fafeb28becb6026395a78fd2065cad23cb70e97466cd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 224176} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23288516, partition_space_available_1k=57801968, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1652836, cpu_share=0.07, mem_res=1444120, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14060, cpu_share=0.0, mem_res=12068, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51440, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9602512, free=734796, buffers=111876, cached=5726456, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.85, five_min_avg=2.53, fifteen_min_avg=2.29, cpu_share=9.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8329d9ddd8c02d3885dd8601585c46be1bbc03e7f64bedb07450b59b740105d2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 225826} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '381fe7c14b3e09c5538995a6bfe8c4a4139150b0cc6c4205506bd9097226fbd1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 225847} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23291164, partition_space_available_1k=57799320, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1657008, cpu_share=0.05, mem_res=1444204, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14076, cpu_share=0.0, mem_res=12084, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51448, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9610104, free=719980, buffers=111988, cached=5733568, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.62, five_min_avg=2.24, fifteen_min_avg=2.2, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5e104d5631f0e72f07dd6eda28d7144bc3dda13b8af8edad76ae71fe2fda0b87', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 227764} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e89d539f41ab9873a87d7b2e07446d7870bba9ec746968ea56bf8c65a64b8fca', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 227786} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23298480, partition_space_available_1k=57792004, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1661500, cpu_share=0.06, mem_res=1444204, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14092, cpu_share=0.0, mem_res=12100, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51456, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9611396, free=709492, buffers=112176, cached=5742576, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.21, five_min_avg=2.02, fifteen_min_avg=2.13, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '77cd460e59287d85b4a3d3bfa5c99bd506a2e7a94c6b1565d7217a9e6d28d3f2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 229713} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '337c6b15a3cab4229ef78ad7b66a67d4b4842f1195b6c701d7e482314a5ab319', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 229734} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23304532, partition_space_available_1k=57785952, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1665492, cpu_share=0.05, mem_res=1444296, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14100, cpu_share=0.0, mem_res=12108, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51464, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9631520, free=677948, buffers=112356, cached=5753816, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.24, five_min_avg=1.89, fifteen_min_avg=2.08, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '97706b4cb90ddc25d3f58d433e3ed51a560829a08ec262e0d244fe8b2dabfd1d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 231823} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '66b36a668703f3b6f5a2d81ab2c7c49d959d4de448233d80c3a414067a9423ca', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 231843} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23321912, partition_space_available_1k=57768572, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1671448, cpu_share=0.09, mem_res=1444232, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14100, cpu_share=0.0, mem_res=12108, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51460, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9653596, free=637064, buffers=112716, cached=5772264, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.33, five_min_avg=2.03, fifteen_min_avg=2.11, cpu_share=11.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '909f8db9ddd6ed2d6658d57da7d7c6a6e7141f693e0ec3c55cd130315b00cdeb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 233806} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '01be934771fb2dbf34a05caf0483f84f1753b59f030438c09c5cb26d6fa9ca68', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 233829} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23325496, partition_space_available_1k=57764988, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1675256, cpu_share=0.06, mem_res=1444308, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14064, cpu_share=0.0, mem_res=12072, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51468, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9646320, free=639688, buffers=112984, cached=5776648, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.21, five_min_avg=2.03, fifteen_min_avg=2.1, cpu_share=9.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bc24b92f81af1cf073a29ebf483b2543c271097f6de8ba1609faae5bfc8df838', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 235460} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd1bd934dbea1fd44c1f44e99b379b8c9eb0e4e203bf198b3cf65b1f6f7b31449', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 235520} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23332640, partition_space_available_1k=57757844, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1678700, cpu_share=0.06, mem_res=1444216, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14084, cpu_share=0.0, mem_res=12092, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51472, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9636840, free=642188, buffers=113168, cached=5783444, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.61, five_min_avg=1.87, fifteen_min_avg=2.04, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2c3bea82b19483f2f6912389a660527003447b6f2b71e2827a6b000a8d799010', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 237411} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'fb863686c24a0b049c1df0cacb22e8079cbdb95b98b57c2b4e777496133dd5af', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 237433} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23334196, partition_space_available_1k=57756288, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1682532, cpu_share=0.05, mem_res=1444532, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14096, cpu_share=0.0, mem_res=12104, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51476, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9641448, free=633692, buffers=113356, cached=5787144, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.41, five_min_avg=1.77, fifteen_min_avg=2.0, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9c52ace4274c401d5271d67e508a98af9fa479ec99d17e688aad8df0990b0d1e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 239369} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '7d5cd53a870bba8b3fd2827d1c825607f2aa184e5ea847ea7759c257d7e052b1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 239393} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23340976, partition_space_available_1k=57749508, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1687788, cpu_share=0.06, mem_res=1445556, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14100, cpu_share=0.0, mem_res=12104, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51480, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9640340, free=627976, buffers=113436, cached=5793888, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.24, five_min_avg=1.67, fifteen_min_avg=1.95, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '21bb18a1650a644817db029de66431878ef89ecb5b9e4f3f0e560dd80842be33', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 241350} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '33bada99d559876d3a8e9202a26f64d7889b324beef8d0f7b596530655c10449', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 241373} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23368692, partition_space_available_1k=57721792, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1695104, cpu_share=0.08, mem_res=1447708, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14116, cpu_share=0.0, mem_res=12120, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51484, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9671616, free=566204, buffers=113748, cached=5824072, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.43, five_min_avg=1.64, fifteen_min_avg=1.92, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '708000e81cbbf302593c78ae82fde62e36e6f9e951bf1d8f7f7caeed95277b8f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 243338} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2f7dad64154fceaf698f90ba5019aa1c910139b4df826a1af16e43dbad8c96f4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 243359} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23381944, partition_space_available_1k=57708540, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1702176, cpu_share=0.08, mem_res=1449588, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14144, cpu_share=0.0, mem_res=12148, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51492, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9656680, free=568052, buffers=114036, cached=5836872, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.1, five_min_avg=1.81, fifteen_min_avg=1.96, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '95400aa3531f71819d850926fd3787302bc6dc5d6c02a03beb0089ede41ee899', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 245106} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1a8a5fa1847174242c8bbebd734644c98063f4ac4bbf919bead55593c72843eb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 245135} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23383672, partition_space_available_1k=57706812, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1706772, cpu_share=0.06, mem_res=1450796, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14100, cpu_share=0.0, mem_res=12104, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51496, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9659224, free=563972, buffers=114168, cached=5838276, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.4, five_min_avg=1.99, fifteen_min_avg=2.01, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4d1b2a97dc823e9ec27f88b66b8176778afb20bc61bc00a9203fa140d4e56fef', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 246972} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '491218b765633db1983419b96ac078bbc148c34ed9adb22eb7165eb51467a8b2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 246994} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23391440, partition_space_available_1k=57699044, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1714036, cpu_share=0.07, mem_res=1453536, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14108, cpu_share=0.0, mem_res=12112, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51500, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9675264, free=537848, buffers=114364, cached=5848164, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.94, five_min_avg=1.94, fifteen_min_avg=1.99, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '814cdf459083fb2ebcba5a2fe6ae8c24bf8495c63e67b3e506743c4e443f9ab7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 248936} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b31fc81b0b45d6de29d2fa2dcc4fc9ecde4f88e03d1b0d3ce7e588bbf20678c3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 248959} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23404172, partition_space_available_1k=57686312, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1852608, cpu_share=0.33, mem_res=1554632, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14124, cpu_share=0.0, mem_res=12128, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51504, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9777888, free=404748, buffers=114836, cached=5878168, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.33, five_min_avg=2.14, fifteen_min_avg=2.06, cpu_share=9.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e52145633c6ec24cdfd3ee9fbbfb9d8dc70d393c578878faf16ee9e9b6c1de49', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 250884} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3167c472ee0f597149d0eeb8091a8d6b1937af46f4df3ec24c7efc33bc306f7a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 250905} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23405784, partition_space_available_1k=57684700, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1854516, cpu_share=0.05, mem_res=1553300, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14128, cpu_share=0.0, mem_res=12132, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51508, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9767476, free=408628, buffers=114960, cached=5884576, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.87, five_min_avg=2.06, fifteen_min_avg=2.04, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a6c499ce03a223f4cc3ac4c62a2aef7cce756dfa7a486f98932149469909a36d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 252876} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '00e3556d47cad9ed5ddace257d6c05607ee7453d75cce407b0ff7700f81e2502', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 252912} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23415200, partition_space_available_1k=57675284, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1859428, cpu_share=0.07, mem_res=1553324, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14132, cpu_share=0.0, mem_res=12136, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51516, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9781464, free=383448, buffers=115200, cached=5895528, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.98, five_min_avg=2.03, fifteen_min_avg=2.03, cpu_share=9.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b36bfc765a26acb9b3f6e0644cb07c2ee7928d451049bf7fbb920dd52e4b737c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 254538} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9bc0db88f7ff52b96af985b8a48197da11b1e4d6a7bc116240a07c4ba1ba6e13', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 254559} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23417048, partition_space_available_1k=57673436, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1864216, cpu_share=0.06, mem_res=1554304, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14140, cpu_share=0.0, mem_res=12144, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51516, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9770112, free=393096, buffers=115328, cached=5897104, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.66, five_min_avg=1.92, fifteen_min_avg=1.99, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '6c516fb754aee59c36f7d068b9c2e4721d67447b622f535bd157fe7879c7d1d5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 256644} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ce26fa0919d86ff3a09a3ecf34c762ebd621fc480625deee104087b6a5fcb34a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 256665} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23429024, partition_space_available_1k=57661460, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1871452, cpu_share=0.07, mem_res=1555620, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=14100, cpu_share=0.0, mem_res=12104, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51528, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9695628, free=377640, buffers=119620, cached=5982752, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.19, five_min_avg=1.98, fifteen_min_avg=2.0, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'cd405f36ae95414f9e873addb3fbc207a06c2deec30c832703d2255942c5af36', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 259372} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1618e46e773331456b9ab02066bd4de3969de458352843ae31d174614c5109ec', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 259431} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23434984, partition_space_available_1k=57655500, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1906000, cpu_share=0.36, mem_res=1592360, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=61828, cpu_share=0.02, mem_res=58572, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51528, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9798208, free=283656, buffers=120076, cached=5973700, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.76, five_min_avg=2.18, fifteen_min_avg=2.07, cpu_share=11.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '58597895394aae11b7a0b0721bfbd649f2658b005fd6449823411688b3dffc4e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 262012} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '536f009834882fd601b0700adcf734cd07ef960385517a1286a95c187e069adb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 262034} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23456204, partition_space_available_1k=57634280, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1909776, cpu_share=0.1, mem_res=1585256, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=61904, cpu_share=0.0, mem_res=58668, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51540, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9765688, free=311628, buffers=119196, cached=5979128, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.43, five_min_avg=2.24, fifteen_min_avg=2.1, cpu_share=11.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dc5e7706fccf004f1050017dd800694dfec21b7bfe8b8a7ecacc901c42d5daa8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 263992} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '4bf920a9efde0f8cb99e2fcab498a45b72679e8986350d56fd93a22c0b366315', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 264014} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23462208, partition_space_available_1k=57628276, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1912544, cpu_share=0.08, mem_res=1584300, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=62028, cpu_share=0.0, mem_res=58764, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51540, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9767572, free=302444, buffers=119376, cached=5986248, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.59, five_min_avg=2.27, fifteen_min_avg=2.12, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'ee9a11681fd9e4a4794da313514bbc2532bf3899e12a1f3ef57ff3d0bab9a7fc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 265656} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8c1beca65f7f4c44623bc9b9e1324c261c2901c36f0c0e879681a17c5e35e712', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 265677} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23469908, partition_space_available_1k=57620576, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1915340, cpu_share=0.06, mem_res=1584224, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=62024, cpu_share=0.0, mem_res=58764, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51548, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9755532, free=312568, buffers=119476, cached=5988064, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.71, five_min_avg=2.05, fifteen_min_avg=2.05, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '031406a465c1ed4822e2f9e77a774d9625384cf465d4dc40b6ee4d274d509051', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 267607} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'de12f99795fdf2444ba4091be330c4b00ab67737d964d21b7522b87896e65b1f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 267629} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23471772, partition_space_available_1k=57618712, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1917972, cpu_share=0.07, mem_res=1584424, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=62024, cpu_share=0.0, mem_res=58764, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51552, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9788492, free=282920, buffers=119592, cached=5984636, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.49, five_min_avg=1.93, fifteen_min_avg=2.01, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '31b2bf7aa5cf5ec1d9eac5da8e2eb2832010631b7bcd1db42f50344ecd7f8dce', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 269541} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'bb56d0e11db6484db0913792ccfab2c566279c584711382a22c7df215071cfdb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 269563} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=23479176, partition_space_available_1k=57611308, percentage_partition_space_used=29, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1918180, cpu_share=0.06, mem_res=1584444, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55248, cpu_share=0.0, mem_res=51988, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51556, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9776252, free=302156, buffers=119144, cached=5978088, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.33, five_min_avg=1.81, fifteen_min_avg=1.96, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f0872adedfceccdc104f85599c5d386d8883473501610995cbd9c1ee0689cf38', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 271606} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '21b2fc60d71a0d3fd256922f34defc899cfef10a345359e79ba42e8631d00ded', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 271628} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24935792, partition_space_available_1k=56154692, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1711708, cpu_share=0.09, mem_res=1584512, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55392, cpu_share=0.0, mem_res=52288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51260, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9759268, free=243096, buffers=77776, cached=6095500, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.5, five_min_avg=2.04, fifteen_min_avg=2.03, cpu_share=12.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'fb88941fbac52e8b75935042884e66adae5a6293ddd10f7d66245efa90677164', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 273534} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '16928ba652f8ef93092e47664cc1de6ec6d9ac0e1752cc1cff5098a034f75a25', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 273580} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24687728, partition_space_available_1k=56402756, percentage_partition_space_used=30, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1691704, cpu_share=0.06, mem_res=1584512, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55440, cpu_share=0.0, mem_res=52336, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51268, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9762732, free=1173704, buffers=78160, cached=5161044, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.54, five_min_avg=2.21, fifteen_min_avg=2.09, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dc7807878254e0e6836a7665ead316ce23f354cb33a117b9ce7c89427467b302', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 275201} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3872da60acf027ab44710999905e6a0b81a2edef367fd55dc2b4e174d514e1c0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 275239} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24694400, partition_space_available_1k=56396084, percentage_partition_space_used=30, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1695464, cpu_share=0.06, mem_res=1584584, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55492, cpu_share=0.0, mem_res=52388, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51276, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9752852, free=1176836, buffers=78308, cached=5167644, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.81, five_min_avg=2.06, fifteen_min_avg=2.05, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c8823d8e79415439607ba70581a135f72f3f96ec2b5ff2dee84489425e8b4f1a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 277184} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '000b8124e93c37be9e5ec55adb9b59b16583b002db61425ed5c4b630471fb61c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 277204} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24700120, partition_space_available_1k=56390364, percentage_partition_space_used=30, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1701176, cpu_share=0.08, mem_res=1584684, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55588, cpu_share=0.0, mem_res=52484, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51280, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9793032, free=1122488, buffers=78740, cached=5181380, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.33, five_min_avg=2.51, fifteen_min_avg=2.21, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0c6fe5f4358d7de7fc05e1aae3e57ebb8c1b3ed22b2cf8aaf2a45372d97f93bb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 279136} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'eea3b8181ea23900e2a4cb9f38df0a56d94c55daeb39cf930d0ae8f65af16a63', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 279157} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24707420, partition_space_available_1k=56383064, percentage_partition_space_used=30, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1705572, cpu_share=0.06, mem_res=1584648, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55648, cpu_share=0.0, mem_res=52544, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51284, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9780444, free=1127464, buffers=78892, cached=5188840, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.74, five_min_avg=2.47, fifteen_min_avg=2.21, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '79ecd1e2cd9456108d33cc1e324853efd70572e04300b6013d7801cc1bc0a640', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 281084} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '20359d3adaf9a44e35bbc35584773b02321663093e1930e4e86b9155676324fb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 281105} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24709160, partition_space_available_1k=56381324, percentage_partition_space_used=30, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1709572, cpu_share=0.05, mem_res=1584676, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55072, cpu_share=0.0, mem_res=51968, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51288, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9772888, free=1128252, buffers=79032, cached=5195468, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.86, five_min_avg=2.29, fifteen_min_avg=2.16, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '296362d77f1ad2cbd3bae61bafb8efd1bd01a28e57eeb7c3577ab5e0450138a2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 283044} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '0569d441e61ba4c0bbc494121907621f8cbe32c98c01fe837f37662d8b7a36d4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 283065} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24712048, partition_space_available_1k=56378436, percentage_partition_space_used=30, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1713968, cpu_share=0.06, mem_res=1584684, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55180, cpu_share=0.0, mem_res=52076, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51292, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9776952, free=1114200, buffers=79188, cached=5205300, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.6, five_min_avg=2.16, fifteen_min_avg=2.13, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b89c42c439022bdf09c1edf1218b8f68bc66b09613bfc90465f07319d43cdc35', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 284738} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '480d0947e26b1322e0492ed5ec86017088cce5906825d72b6a0c8b8cdd6c73a9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 284760} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24747484, partition_space_available_1k=56343000, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1746344, cpu_share=0.13, mem_res=1581416, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55240, cpu_share=0.0, mem_res=52136, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51304, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9767392, free=1082872, buffers=79664, cached=5245712, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.71, five_min_avg=2.08, fifteen_min_avg=2.1, cpu_share=9.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b176e742d5d4a3118de0660717271b658dc75a38a6d43b0df420e17620488126', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 286690} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ff94f665ff37f30204bd0d9f1ba20ae44b4461d9b635b9678d923b53ef490473', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 286712} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24754164, partition_space_available_1k=56336320, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1749476, cpu_share=0.06, mem_res=1580788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55244, cpu_share=0.0, mem_res=52136, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51304, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9783164, free=1060204, buffers=79808, cached=5252464, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.44, five_min_avg=1.94, fifteen_min_avg=2.05, cpu_share=7.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dbbdd9096ae00a1ddb3e17591162df711392512597fe761dc7c81f7119dc214f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 288645} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'fa42461ad011a94727a3a337e5c0bc13426dba725014671fd0e561954b4f0445', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 288670} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24756464, partition_space_available_1k=56334020, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1753668, cpu_share=0.06, mem_res=1580788, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55300, cpu_share=0.0, mem_res=52192, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51304, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9793960, free=1044548, buffers=79988, cached=5257144, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.51, five_min_avg=1.86, fifteen_min_avg=2.01, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e1e9a7898b3aea18da74c317277c9c864c3139ac0d360a15c004c70f32e2c3ce', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 290701} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1e0bf0b137ed9647134efa7e22c22baa277e6ae86d72933b15bd856b44fcc60d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 290722} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24770208, partition_space_available_1k=56320276, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1759588, cpu_share=0.08, mem_res=1580676, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55304, cpu_share=0.0, mem_res=52196, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51312, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9796692, free=1027228, buffers=80336, cached=5271384, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.14, five_min_avg=2.23, fifteen_min_avg=2.12, cpu_share=11.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0267fd47c3792b3b35bddae325be2ffe8bb341c1ead1357e98f266ff65b27b1e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 292661} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '85a44d5041114824df972f74cfec596925e4fcdd7f2cd23d9ff4b3d14d1e5cdf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 292680} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24779192, partition_space_available_1k=56311292, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1764188, cpu_share=0.06, mem_res=1580768, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55160, cpu_share=0.0, mem_res=52052, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51320, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9786208, free=1022936, buffers=80460, cached=5286036, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.2, five_min_avg=2.14, fifteen_min_avg=2.1, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '432d6f41e904441bedd99ce0d7e84dc4718c0ddc8ea829b939b8cdf1847e81f2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 294314} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6cda8c57fced9349327118cba1bc705685b97d6812baf73c79d44b637f763fcd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 294335} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24786164, partition_space_available_1k=56304320, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1769444, cpu_share=0.06, mem_res=1582012, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55184, cpu_share=0.0, mem_res=52076, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51320, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9780344, free=1019584, buffers=80644, cached=5295068, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=1.99, fifteen_min_avg=2.05, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a122ec9b8988a24b120fd4800f9151d4a0448aed84f6dca49750ed897b54145c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 296265} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5c9434c39e722e7b1f07373d4f79e027e02e25a42abec27c40a517e28b798032', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 296287} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24794116, partition_space_available_1k=56296368, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1773932, cpu_share=0.06, mem_res=1583556, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55184, cpu_share=0.0, mem_res=52076, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51328, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9792808, free=993620, buffers=80800, cached=5308412, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.85, five_min_avg=1.99, fifteen_min_avg=2.04, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e718577367c201b7cd4bb4df85418a0c7c06ad9003c7e7ac31be30fb7496c027', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 298298} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e69186725ad4bc8a1c7cd96410faaf754a6e9a6f0ea521ad4193028f7a0c9795', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 298320} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24800868, partition_space_available_1k=56289616, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1778820, cpu_share=0.05, mem_res=1584760, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55288, cpu_share=0.0, mem_res=52180, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51332, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9813704, free=956464, buffers=81156, cached=5324316, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.39, five_min_avg=1.84, fifteen_min_avg=1.99, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '89786d95a619c41b7cbfd4ffb4642c5559f6c862d612fe0bf5cb4be455ab5060', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 300342} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '793893e43ce5a493b2d796446bcaaaf41a813c76974aaa1b27fea2f9d2983cc5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 300364} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24821084, partition_space_available_1k=56269400, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1792436, cpu_share=0.11, mem_res=1587944, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55304, cpu_share=0.0, mem_res=52196, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51332, cpu_share=0.01, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9854928, free=894712, buffers=81744, cached=5344256, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.18, five_min_avg=2.01, fifteen_min_avg=2.04, cpu_share=12.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b93288ec605130c5b155fd41fec97aa847c6574211cbf5ee553bb093056b24e0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 302337} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8273111c83713daef6843ef76eba399943e6bfee023ae028543b15a943917043', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 302361} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24838728, partition_space_available_1k=56251756, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1808064, cpu_share=0.13, mem_res=1592448, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55304, cpu_share=0.0, mem_res=52196, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51340, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9837724, free=888928, buffers=82264, cached=5366724, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.04, five_min_avg=2.3, fifteen_min_avg=2.13, cpu_share=12.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f8921f3e8cc8bf0145765aa9dbd6a052e31e71b746ad3aa9f26e8adb56ff4ac5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 304064} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'a6dddaa5d3e690926c3cd5cfc5df4c892dbd069319a7253a5a98b66b86d1f50d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 304086} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24852088, partition_space_available_1k=56238396, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1815276, cpu_share=0.08, mem_res=1594820, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55156, cpu_share=0.0, mem_res=52048, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51344, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=10038040, free=609604, buffers=82704, cached=5445292, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.3, five_min_avg=2.93, fifteen_min_avg=2.35, cpu_share=11.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '241e2d13597c22745df18d21d0555d916b143ac943453daf29aa6db7f37c5a65', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 306118} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2a3402b94d2741f2fd917cb1db4b82efbe55790dffe3f6ece93256f528a8d5c8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 306141} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24864608, partition_space_available_1k=56225876, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2004848, cpu_share=0.15, mem_res=1749300, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=55216, cpu_share=0.0, mem_res=52108, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51352, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=10018456, free=660716, buffers=83076, cached=5413392, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.04, five_min_avg=3.05, fifteen_min_avg=2.43, cpu_share=10.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 03:58:29] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6759 0.001571 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4550fde2d3a4fb33372ea6ba65123bc57b51462c55da64d93e513811a5b0cc89', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 308165} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c285daea1072cecb59ab841d337822337b19a9916d40372c0a0423c9d6052132', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 308185} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24882724, partition_space_available_1k=56207760, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2008604, cpu_share=0.07, mem_res=1747956, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56804, cpu_share=0.0, mem_res=52224, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51356, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=10005320, free=656548, buffers=83476, cached=5430296, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.46, five_min_avg=2.81, fifteen_min_avg=2.39, cpu_share=9.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c994de4912540a2b390c6d18563758847273b4899d603f182575e05cfca03a79', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 310112} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b2f338fe93c77b9125599f9605d5c89ee48d2dddb9983739e755ceb001bda561', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 310134} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24884196, partition_space_available_1k=56206288, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2012360, cpu_share=0.06, mem_res=1747952, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56856, cpu_share=0.0, mem_res=52276, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51360, cpu_share=0.02, mem_res=45264, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=10000536, free=659576, buffers=83612, cached=5431916, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.16, five_min_avg=2.66, fifteen_min_avg=2.36, cpu_share=7.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8b1b6f770342d1488f6cfce760e06189066dd50beb3ac89a8e1ba4d3734bca3f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 311968} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8e9c37d859c09f5021551d426458bfa4857a4037dfae41a2c124055f4574ee27', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 312028} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24901108, partition_space_available_1k=56189376, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2016124, cpu_share=0.06, mem_res=1747820, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56908, cpu_share=0.0, mem_res=52328, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51368, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9995508, free=646280, buffers=83828, cached=5450024, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.76, five_min_avg=2.47, fifteen_min_avg=2.32, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7e4376a602c27c1b1d0ae07bd31736457a0d3c7f66adfa255189caf40fdde469', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 313799} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '085f18822b16c479cb5a65f50061bb147eb972daa25010a23af1652ec9f695f7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 313821} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24905308, partition_space_available_1k=56185176, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2021316, cpu_share=0.07, mem_res=1748980, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56956, cpu_share=0.0, mem_res=52376, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51380, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=10010460, free=625288, buffers=84048, cached=5455844, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.7, five_min_avg=2.32, fifteen_min_avg=2.27, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f2ed51f5366efcf5237a2d1ab8a3e24add8ce5aa9f0fc8fc4b49f41826ba09ef', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 316015} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '23f1d6c587fee0bd0e626595aef9fbdfc9c7eebcbd0aaa905215c4903752b627', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 316037} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24912792, partition_space_available_1k=56177692, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2025172, cpu_share=0.06, mem_res=1749048, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56616, cpu_share=0.0, mem_res=52040, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51388, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9998968, free=601644, buffers=85892, cached=5489136, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.64, five_min_avg=2.19, fifteen_min_avg=2.23, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 04:04:14] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6757 0.001378 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0d540b8968fc17031776563e1826edbeaf91faf48e2a286f4b4e6483e92ff0fc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 318251} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'acff6cd24cefa485d377a91afb7cfaa5ca68b93e04f28770a161046765f2cb25', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 318271} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24917976, partition_space_available_1k=56172508, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2032344, cpu_share=0.07, mem_res=1751056, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56772, cpu_share=0.0, mem_res=52192, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51384, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9885476, free=677928, buffers=87476, cached=5524760, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.17, five_min_avg=2.24, fifteen_min_avg=2.25, cpu_share=10.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a90f347c45c987093ccdf5a54ba28190e4329512bd1bcd0f7c1c02245ca8b590', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 320205} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'edd655d7f7c28521c13af06e401643e318585b0e435e0e0fa2392011b2cf48a5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 320226} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24927984, partition_space_available_1k=56162500, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2037772, cpu_share=0.06, mem_res=1752308, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56824, cpu_share=0.0, mem_res=52244, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51396, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9882416, free=671020, buffers=87652, cached=5534552, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.41, five_min_avg=2.39, fifteen_min_avg=2.3, cpu_share=8.75, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '2302900dd6f511cb65d5dfbcc8dc62ee2b2cbecfec392186633bdcc9996f006c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 322026} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'd71a76db2de3f0dac694a2ba64468cb021c7e2ec6b07ef4a7862de55e9dfcc10', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 322048} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24939220, partition_space_available_1k=56151264, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2057292, cpu_share=0.08, mem_res=1753920, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56884, cpu_share=0.0, mem_res=52304, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51400, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9868772, free=652992, buffers=87976, cached=5565900, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.62, five_min_avg=2.18, fifteen_min_avg=2.23, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '80a9a58d4025a70e454c5ff0bed5bd8baf7ae0caa3f0d50d01487b70470ecedf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 323878} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ca0e3f99527a6c0a808afaec9e106da3569367e4eeb6fb51fe420127a6471395', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 323899} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24971484, partition_space_available_1k=56119000, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2052752, cpu_share=0.08, mem_res=1731676, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56996, cpu_share=0.0, mem_res=52416, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51408, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9884004, free=624584, buffers=88140, cached=5578912, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.42, five_min_avg=2.02, fifteen_min_avg=2.18, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '39249b329c473533192f253b08af3063becf8009239f32303c3262adcb592650', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 325870} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2ba3cf453e1472e0c585f2343e07e96eb4a920803d05f4e1e6ce264b200fac6f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 325892} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24979572, partition_space_available_1k=56110912, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2057452, cpu_share=0.07, mem_res=1732112, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=57044, cpu_share=0.0, mem_res=52464, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51408, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9883600, free=615864, buffers=88496, cached=5587680, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.8, five_min_avg=2.07, fifteen_min_avg=2.18, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'dac4833688b68fd88b5cca059d80e37e1af46ad24a3f20ac5d79203c63849f86', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 327889} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '069404c30c744a2515844ce9dad38ba785562f2802334c263ba744fb1b1088f7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 327911} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24991656, partition_space_available_1k=56098828, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2061600, cpu_share=0.08, mem_res=1731396, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56648, cpu_share=0.0, mem_res=52068, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51416, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9881808, free=605112, buffers=88724, cached=5599996, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.68, five_min_avg=1.98, fifteen_min_avg=2.14, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5d6845971a8964fcb31cba1576a59f3ca1bac7fd71dcce7d031ec4725b2d9c57', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 329847} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c9d41221d352ca4ef15fb2c78c4fc0a9f5fe6bf355d254b8076b1addc2d2e398', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 329870} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=24998384, partition_space_available_1k=56092100, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2065520, cpu_share=0.05, mem_res=1731544, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56764, cpu_share=0.0, mem_res=52184, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51420, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9879092, free=605720, buffers=88852, cached=5601976, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.69, five_min_avg=1.91, fifteen_min_avg=2.11, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7e719bca445a143182251b2bc9a57533de97e9a86acafd50698c94e730232c7a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 331501} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8d5e83aa887222dda1e8ae1425843a7dd22ba461041bbd38e6f1bacbd02329d2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 331523} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25000108, partition_space_available_1k=56090376, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2068992, cpu_share=0.06, mem_res=1731552, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56812, cpu_share=0.0, mem_res=52232, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51424, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9871464, free=605720, buffers=88968, cached=5609488, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.12, five_min_avg=1.71, fifteen_min_avg=2.02, cpu_share=8.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8e20671f85e77f5a26f57b625f1d50a8dcb1c9ebfcc8f2d850b4177bb901b4b7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 333463} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6b89f7f2258d746f46790ecd7ebc85ad9d77526c751e191f8c46eb093d1747d7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 333485} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25007236, partition_space_available_1k=56083248, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2072652, cpu_share=0.06, mem_res=1731612, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56868, cpu_share=0.0, mem_res=52288, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51428, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9901732, free=568648, buffers=89076, cached=5616184, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.56, five_min_avg=1.74, fifteen_min_avg=2.01, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1ae21fad3fb2d652edea74aae5531e99f4cc5fcd77eb906e9730a13af452596f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 335412} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '8b4579254c37e96440fdde818e5f85ff446015366383faab8dc130eaa42df9bd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 335433} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25003176, partition_space_available_1k=56087308, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2076096, cpu_share=0.06, mem_res=1731832, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56924, cpu_share=0.0, mem_res=52344, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51432, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9896148, free=578000, buffers=89184, cached=5612308, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.73, five_min_avg=1.78, fifteen_min_avg=2.01, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '82ce3532a1e6b060600300da26bab79fd73a9db56791619b9e73441a0c11eeb9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 337422} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '081c815692c7574ad9fd97627056855d9ddaecca5cf2d8a33ac731e0a6a8a20d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 337443} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25019656, partition_space_available_1k=56070828, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2084876, cpu_share=0.07, mem_res=1734428, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=57032, cpu_share=0.0, mem_res=52452, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51440, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9922224, free=534412, buffers=89572, cached=5629432, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.31, five_min_avg=2.18, fifteen_min_avg=2.13, cpu_share=10.5, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9b2b5d064aa8553afe8a4b2056783e0faacc551cdc8edee17edd8f310746c27d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 339379} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '3062540936f92be28a9210a5803899441a590933ce94809b134c46e46a1ef249', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 339400} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25020924, partition_space_available_1k=56069560, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2090264, cpu_share=0.05, mem_res=1736204, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56660, cpu_share=0.0, mem_res=52080, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51444, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9898396, free=556584, buffers=89672, cached=5630988, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.3, five_min_avg=2.09, fifteen_min_avg=2.1, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'f1b2500cd4ed20f58b1c5e4994cc2ad9fc6ce00dabbcf4b6220b26d8ee7289bb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 341041} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2c70b0c24859f7595cbc67bce94f9669aeaad57b02ca0b4b34769e456ee712e6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 341062} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25027652, partition_space_available_1k=56062832, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2095292, cpu_share=0.06, mem_res=1737484, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56668, cpu_share=0.0, mem_res=52088, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51448, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9897976, free=549640, buffers=89764, cached=5638260, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.59, five_min_avg=1.91, fifteen_min_avg=2.03, cpu_share=8.25, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'a86ffbeaf4ecc8a1500fc4cc1c597c0140ba1b937c39a728c881dce57bc969e5', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 343007} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '21124642e1d7989695282bce42932501ded5482350341b0a00e1ad6105fa284d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 343029} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25029096, partition_space_available_1k=56061388, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2099488, cpu_share=0.06, mem_res=1738512, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56780, cpu_share=0.0, mem_res=52200, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51452, cpu_share=0.01, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9914740, free=531864, buffers=89896, cached=5639140, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.53, five_min_avg=1.83, fifteen_min_avg=1.99, cpu_share=8.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '4cad2be431e9463e15cecb23bed1376553038c9ed77f0237427ba5cad91147b3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 345574} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b0c113ec87bf3742dc8af114274e0d747e1ff98def6c88d467f745232c86b114', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 345602} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: DEBUG:contrail-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-ansible-os-3956-1.', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=25226428, partition_space_available_1k=55864056, percentage_partition_space_used=31, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=2093012, cpu_share=0.07, mem_res=1741480, _EventManager__key='cassandra'), 'contrail-query-engine': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=56824, cpu_share=0.0, mem_res=52244, _EventManager__key='contrail-query-engine'), 'contrail-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=51456, cpu_share=0.02, mem_res=45272, _EventManager__key='contrail-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=16175640, used=9738144, free=391256, buffers=97212, cached=5949028, node_type='database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=1.49, five_min_avg=1.76, fifteen_min_avg=1.96, cpu_share=10.0, node_type='database-node'), running_package_version=None, installed_package_version=None, _table='ObjectDatabaseInfo') DEBUG:contrail-database-nodemgr:send sync_uve in the [ObjectDatabaseInfo:NodeStatusUVE] map [Introspect:8103]10.0.0.19 - - [2025-08-28 04:19:16] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6757 0.001475 DEBUG:contrail-database-nodemgr:dir for analytics is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'da5ae5b23ba45899e9038c915f445999d05eab510f3f59880ea20c839ac3d567', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['10.0.0.19', '9042', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 348627} WARNING:contrail-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2224df336b4d3fc9465a6b5bdc3d4354b3c09d8cd4d742df8da98b25823835b2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7200', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': '9b21724bbfa7d362b8bff40b74bed77b3814b04b58fc207ca945b359901dcf17', 'DetachKeys': '', 'Pid': 348715} ERROR:contrail-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 Exception ignored in: > Traceback (most recent call last): File "/usr/local/lib64/python3.9/site-packages/gevent/threading.py", line 398, in after_fork_in_child assert not thread.is_alive() File "/usr/lib64/python3.9/threading.py", line 1349, in is_alive assert not self._is_stopped and self._started.is_set() AssertionError: