++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/config-svc-monitor ++ CONTAINER_LOG_DIR=/var/log/contrail/config-svc-monitor ++ mkdir -p /var/log/contrail/config-svc-monitor ++ log_file=/var/log/contrail/config-svc-monitor/console.log ++ touch /var/log/contrail/config-svc-monitor/console.log ++ chmod 600 /var/log/contrail/config-svc-monitor/console.log ++ exec +++ tee -a /var/log/contrail/config-svc-monitor/console.log +++ date ++ echo 'INFO: =================== Mon Mar 2 02:05:05 AM UTC 2026 ===================' INFO: =================== Mon Mar 2 02:05:05 AM UTC 2026 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' +++++ ip route get 1 ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ cut -d / -f 1 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ grep 'inet ' +++ head -n 1 +++ awk '{print $2}' +++ ip addr show dev ens3 ++ DEFAULT_LOCAL_IP=10.0.0.27 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=256 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 256 ]] ++ [[ -z '' ]] +++ awk '/pagesize=2M/{print($3)}' +++ mount -t hugetlbfs +++ tail -n 1 ++ HUGE_PAGES_2MB_DIR= ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=openstack ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=rbac ++ AUTH_MODE=keystone ++ AUTH_PARAMS= ++ SSL_ENABLE=false ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICS_ALARM_ENABLE=True ++ ANALYTICS_SNMP_ENABLE=True ++ ANALYTICSDB_ENABLE=True ++ ANALYTICS_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICSDB_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICS_SNMP_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:8081 ' +++ '[' -n '10.0.0.242:8081 10.0.0.27:8081 10.0.0.23:8081 ' ']' +++ echo '10.0.0.242:8081 10.0.0.27:8081 10.0.0.23:8081' ++ ANALYTICS_SERVERS='10.0.0.242:8081 10.0.0.27:8081 10.0.0.23:8081' +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9042 ' +++ '[' -n '10.0.0.242:9042 10.0.0.27:9042 10.0.0.23:9042 ' ']' +++ echo '10.0.0.242:9042 10.0.0.27:9042 10.0.0.23:9042' ++ ANALYTICSDB_CQL_SERVERS='10.0.0.242:9042 10.0.0.27:9042 10.0.0.23:9042' ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:8086 ' +++ '[' -n '10.0.0.242:8086 10.0.0.27:8086 10.0.0.23:8086 ' ']' +++ echo '10.0.0.242:8086 10.0.0.27:8086 10.0.0.23:8086' ++ COLLECTOR_SERVERS='10.0.0.242:8086 10.0.0.27:8086 10.0.0.23:8086' ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ CONFIGDB_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:8082 ' +++ '[' -n '10.0.0.242:8082 10.0.0.27:8082 10.0.0.23:8082 ' ']' +++ echo '10.0.0.242:8082 10.0.0.27:8082 10.0.0.23:8082' ++ CONFIG_SERVERS='10.0.0.242:8082 10.0.0.27:8082 10.0.0.23:8082' +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9161 ' +++ '[' -n '10.0.0.242:9161 10.0.0.27:9161 10.0.0.23:9161 ' ']' +++ echo '10.0.0.242:9161 10.0.0.27:9161 10.0.0.23:9161' ++ CONFIGDB_SERVERS='10.0.0.242:9161 10.0.0.27:9161 10.0.0.23:9161' +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9041 ' +++ '[' -n '10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041 ' ']' +++ echo '10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041' ++ CONFIGDB_CQL_SERVERS='10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041' ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=false ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=false ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=false ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.20.0.193,10.20.0.254,10.20.0.14 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.20.0.193,10.20.0.254,10.20.0.14 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+=10.0.0.242:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+=10.0.0.27:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+=10.0.0.23:2181, +++ '[' -n 10.0.0.242:2181,10.0.0.27:2181,10.0.0.23:2181, ']' +++ echo 10.0.0.242:2181,10.0.0.27:2181,10.0.0.23:2181 ++ ZOOKEEPER_SERVERS=10.0.0.242:2181,10.0.0.27:2181,10.0.0.23:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:2181 ' +++ '[' -n '10.0.0.242:2181 10.0.0.27:2181 10.0.0.23:2181 ' ']' +++ echo '10.0.0.242:2181 10.0.0.27:2181 10.0.0.23:2181' ++ ZOOKEEPER_SERVERS_SPACE_DELIM='10.0.0.242:2181 10.0.0.27:2181 10.0.0.23:2181' ++ RABBITMQ_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+=10.0.0.242:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+=10.0.0.27:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+=10.0.0.23:5673, +++ '[' -n 10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673, ']' +++ echo 10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673 ++ RABBITMQ_SERVERS=10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=false ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:6379 ' +++ '[' -n '10.0.0.242:6379 10.0.0.27:6379 10.0.0.23:6379 ' ']' +++ echo '10.0.0.242:6379 10.0.0.27:6379 10.0.0.23:6379' ++ REDIS_SERVERS='10.0.0.242:6379 10.0.0.27:6379 10.0.0.23:6379' ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=false ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9092 ' +++ '[' -n '10.0.0.242:9092 10.0.0.27:9092 10.0.0.23:9092 ' ']' +++ echo '10.0.0.242:9092 10.0.0.27:9092 10.0.0.23:9092' ++ KAFKA_SERVERS='10.0.0.242:9092 10.0.0.27:9092 10.0.0.23:9092' ++ KAFKA_SSL_ENABLE=false ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=10.0.0.242 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=false ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=false ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=false ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ /v3 == \/\v\2\.\0 ]] ++ [[ openstack == \o\p\e\n\s\t\a\c\k ]] ++ AUTH_MODE=keystone ++ [[ keystone == \k\e\y\s\t\o\n\e ]] ++ AUTH_PARAMS='--admin_password contrail123' ++ AUTH_PARAMS+=' --admin_tenant_name admin' ++ AUTH_PARAMS+=' --admin_user admin' ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + wait_for_cassandra ++ echo 10.0.0.242,10.0.0.27,10.0.0.23 ++ cut -d , -f 1 + local cassandra=10.0.0.242 + (( i=1 )) + (( i<=30 )) + nc -z 10.0.0.242 9041 + echo -e '\035\nquit' + echo 'INFO: Cassandra is connected' INFO: Cassandra is connected + sleep 5 + return 0 ++ get_listen_ip_for_node CONFIG +++ find_my_ip_and_order_for_node CONFIG +++ local server_typ=CONFIG_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.242,10.0.0.27,10.0.0.23 +++ local servers=10.0.0.242,10.0.0.27,10.0.0.23 +++ local server_list= +++ IFS=, +++ read -ra server_list ++++ get_local_ips +++ cut -d ' ' -f 1 ++++ awk '/32 host/ { print f } {f=$2}' ++++ uniq ++++ tr '\n' , ++++ cat /proc/net/fib_trie ++++ grep -vi host ++++ sort +++ local local_ips=,10.0.0.27,10.20.0.254,127.0.0.1,172.17.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.242'\''))' +++ local server_ip=10.0.0.242 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.242 ]] +++ [[ ,10.0.0.27,10.20.0.254,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.242, ]] +++ (( ord+=1 )) +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.27'\''))' +++ local server_ip=10.0.0.27 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.27 ]] +++ [[ ,10.0.0.27,10.20.0.254,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.27, ]] +++ echo 10.0.0.27 2 +++ return ++ local ip=10.0.0.27 ++ [[ -z 10.0.0.27 ]] ++ echo 10.0.0.27 + host_ip=10.0.0.27 + [[ cql == \c\q\l ]] ++ echo 10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041 ++ sed 's/,/ /g' + cassandra_server_list='10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041' + mkdir -p /etc/contrail + cat ++ get_introspect_listen_ip_for_node CONFIG ++ local ip=0.0.0.0 ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ echo 0.0.0.0 + add_ini_params_from_env SVC_MONITOR /etc/contrail/contrail-svc-monitor.conf + local service_name=SVC_MONITOR + local cfg_path=/etc/contrail/contrail-svc-monitor.conf + local delim=__ ++ grep '^SVC_MONITOR__.*__.*=.*$' ++ sort ++ sed 's/^SVC_MONITOR__//g' ++ cut -d = -f 1 ++ set -o posix ++ set + local vars= + local section= + set_third_party_auth_config + [[ keystone != \k\e\y\s\t\o\n\e ]] + local tmp_file=/etc/contrail/contrail-keystone-auth.conf.tmp + cat + [[ /v3 == \/\v\3 ]] + cat + [[ http == \h\t\t\p\s ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n public ]] + echo 'interface = public' + mv -f /etc/contrail/contrail-keystone-auth.conf.tmp /etc/contrail/contrail-keystone-auth.conf + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ keystone == \k\e\y\s\t\o\n\e ]] + cat + [[ http == \h\t\t\p\s ]] + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + upgrade_old_logs svc-monitor + local template=svc-monitor ++ dirname /var/log/contrail/config-svc-monitor + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/config-svc-monitor + chmod 755 /var/log/contrail/config-svc-monitor + mv -n '/var/log/contrail/*svc-monitor.log*' /var/log/contrail/config-svc-monitor/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/config-svc-monitor + find /var/log/contrail/config-svc-monitor -uid 0 -exec chown 1999:1999 '{}' + + run_service /usr/local/bin/contrail-svc-monitor --conf_file /etc/contrail/contrail-svc-monitor.conf --conf_file /etc/contrail/contrail-keystone-auth.conf + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + mkdir -p /etc/contrail /var/lib/contrail + chown 1999:1999 /etc/contrail /var/lib/contrail + find /etc/contrail -uid 0 -exec chown 1999:1999 '{}' + + chmod 755 /etc/contrail + do_run_service /usr/local/bin/contrail-svc-monitor --conf_file /etc/contrail/contrail-svc-monitor.conf --conf_file /etc/contrail/contrail-keystone-auth.conf + [[ -n 1999 ]] + [[ -n 1999 ]] + mkdir -p /var/crashes + chmod 777 /var/crashes ++ id -un 1999 + local user_name=contrail + export HOME=/home/contrail + HOME=/home/contrail + mkdir -p /home/contrail + chown -R 1999:1999 /home/contrail + exec setpriv --reuid 1999 --regid 1999 --clear-groups --no-new-privs /usr/local/bin/contrail-svc-monitor --conf_file /etc/contrail/contrail-svc-monitor.conf --conf_file /etc/contrail/contrail-keystone-auth.conf 03/02/2026 02:05:14.236 7f4a60c8a3c0 [contrail-svc-monitor] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 03/02/2026 02:05:14.403 7f4a60c8a3c0 [contrail-svc-monitor] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 03/02/2026 02:05:14.403 7f4a60c8a3c0 [contrail-svc-monitor] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/config-svc-monitor/contrail-svc-monitor.log] InternalError Python 3.9.25: /usr/bin/python3 Mon Mar 2 02:15:14 2026 A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred. /usr/local/bin/contrail-svc-monitor in 4 import sys 5 from svc_monitor.svc_monitor import server_main 6 if __name__ == '__main__': 7 sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 8 sys.exit(server_main()) sys = sys.exit = server_main = /usr/local/lib/python3.9/site-packages/svc_monitor/svc_monitor.py in server_main() 1009 def server_main(): 1010 vnc_cgitb.enable(format='text') 1011 main() 1012 # end server_main 1013 global main = /usr/local/lib/python3.9/site-packages/svc_monitor/svc_monitor.py in main(args_str='--conf_file /etc/contrail/contrail-svc-monitor.c...nf_file /etc/contrail/contrail-keystone-auth.conf') 994 REACTION_MAP, 'svc_monitor', rabbitmq_cfg, 995 host_ip, args.trace_file) 996 vnc_amqp.establish() 997 vnc_amqp.close() 998 sm_logger.debug("Removed remained AMQP queue") vnc_amqp = vnc_amqp.establish = > /usr/local/lib/python3.9/site-packages/cfgm_common/vnc_amqp.py in establish(self=) 33 def establish(self): 34 q_name = '.'.join([self.q_name_prefix, socket.getfqdn(self.host_ip)]) 35 self._vnc_kombu = VncKombuClient( 36 self._rabbitmq_cfg['servers'], self._rabbitmq_cfg['port'], 37 self._rabbitmq_cfg['user'], self._rabbitmq_cfg['password'], self = self._vnc_kombu = None global VncKombuClient = self._rabbitmq_cfg = {'ha_mode': False, 'password': '***', 'port': '5672', 'servers': '10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673', 'ssl_ca_certs': '', 'ssl_certfile': '', 'ssl_keyfile': '', 'ssl_version': '', 'use_ssl': 'false', 'user': 'guest', ...} q_name = 'svc_monitor.cn-jenkins-deploy-platform-ansible-os-5413-2.' self._vnc_subscribe_callback = > self.logger = self.logger.log = > rabbit_use_ssl undefined kombu_ssl_version undefined kombu_ssl_keyfile undefined kombu_ssl_certfile undefined kombu_ssl_ca_certs undefined register_handler undefined self.register_handler = True /usr/local/lib/python3.9/site-packages/cfgm_common/vnc_kombu.py in __init__(self=, rabbit_hosts='10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673', rabbit_port='5672', rabbit_user='guest', rabbit_password='***', rabbit_vhost='/', rabbit_ha_mode=False, q_name='svc_monitor.cn-jenkins-deploy-platform-ansible-os-5413-2.', subscribe_cb=>, logger=>, heartbeat_seconds=0, **kwargs={'kombu_ssl_ca_certs': '', 'kombu_ssl_certfile': '', 'kombu_ssl_keyfile': '', 'kombu_ssl_version': '', 'rabbit_use_ssl': 'false', 'register_handler': True}) 366 durable=False, 367 queue_arguments=queue_args) 368 self._start(q_name) 369 # end __init__ 370 self = self._start = > q_name = 'svc_monitor.cn-jenkins-deploy-platform-ansible-os-5413-2.' /usr/local/lib/python3.9/site-packages/cfgm_common/vnc_kombu.py in _start(self=, client_name='svc_monitor.cn-jenkins-deploy-platform-ansible-os-5413-2.') 221 def _start(self, client_name): 222 self._running = True 223 self._reconnect_drain(delete_old_q=True) 224 self._reconnect_publish() 225 self = self._reconnect_drain = > delete_old_q undefined /usr/local/lib/python3.9/site-packages/cfgm_common/vnc_kombu.py in _reconnect_drain(self=, delete_old_q=True) 99 self._logger(msg, level=SandeshLevel.SYS_ERR) 100 101 self._consumer = kombu.Consumer(self._channel_drain, 102 queues=self._update_queue_obj, 103 callbacks=[self._subscribe]) self = self._consumer undefined global kombu = kombu.Consumer = self._channel_drain = queues undefined self._update_queue_obj = -> > callbacks undefined self._subscribe = > /usr/local/lib/python3.9/site-packages/kombu/messaging.py in __init__(self= -> bound to chan:None>]>, channel=, queues= -> >, no_ack=None, auto_declare=None, callbacks=[>], on_decode_error=None, on_message=None, accept=None, prefetch_count=None, tag_prefix=None) 412 413 if self.channel: 414 self.revive(self.channel) 415 416 @property self = -> bound to chan:None>]> self.revive = -> bound to chan:None>]>> self.channel = /usr/local/lib/python3.9/site-packages/kombu/messaging.py in revive(self= -> bound to chan:None>]>, channel=) 434 435 if self.auto_declare: 436 self.declare() 437 438 if self.prefetch_count is not None: self = -> bound to chan:None>]> self.declare = -> bound to chan:None>]>> /usr/local/lib/python3.9/site-packages/kombu/messaging.py in declare(self= -> bound to chan:None>]>) 448 """ 449 for queue in self._queues.values(): 450 queue.declare() 451 452 def register_callback(self, callback): queue = -> bound to chan:None> queue.declare = -> bound to chan:None>> /usr/local/lib/python3.9/site-packages/kombu/entity.py in declare(self= -> bound to chan:None>, nowait=False, channel=None) 615 # - declare main binding. 616 self._create_exchange(nowait=nowait, channel=channel) 617 self._create_queue(nowait=nowait, channel=channel) 618 self._create_bindings(nowait=nowait, channel=channel) 619 return self.name self = -> bound to chan:None> self._create_queue = -> bound to chan:None>> nowait = False channel = None /usr/local/lib/python3.9/site-packages/kombu/entity.py in _create_queue(self= -> bound to chan:None>, nowait=False, channel=None) 624 625 def _create_queue(self, nowait=False, channel=None): 626 self.queue_declare(nowait=nowait, passive=False, channel=channel) 627 if self.exchange and self.exchange.name: 628 self.queue_bind(nowait=nowait, channel=channel) self = -> bound to chan:None> self.queue_declare = -> bound to chan:None>> nowait = False passive undefined channel = None /usr/local/lib/python3.9/site-packages/kombu/entity.py in queue_declare(self= -> bound to chan:None>, nowait=False, passive=False, channel=) 653 max_priority=self.max_priority, 654 ) 655 ret = channel.queue_declare( 656 queue=self.name, 657 passive=passive, ret undefined channel = channel.queue_declare = > queue undefined self = -> bound to chan:None> self.name = 'svc_monitor.cn-jenkins-deploy-platform-ansible-os-5413-2.' passive = False durable undefined self.durable = False exclusive undefined self.exclusive = False auto_delete undefined self.auto_delete = False arguments undefined queue_arguments = {} nowait = False /usr/local/lib/python3.9/site-packages/amqp/channel.py in queue_declare(self=, queue='svc_monitor.cn-jenkins-deploy-platform-ansible-os-5413-2.', passive=False, durable=False, exclusive=False, auto_delete=False, nowait=False, arguments={}, argsig='BsbbbbbF') 1160 ) 1161 if not nowait: 1162 return queue_declare_ok_t(*self.wait( 1163 spec.Queue.DeclareOk, returns_tuple=True, 1164 )) global queue_declare_ok_t = self = self.wait = > global spec = spec.Queue = spec.Queue.DeclareOk = (50, 11) returns_tuple undefined /usr/local/lib/python3.9/site-packages/amqp/abstract_channel.py in wait(self=, method=[(50, 11)], callback=None, timeout=None, returns_tuple=True) 97 try: 98 while not p.ready: 99 self.connection.drain_events(timeout=timeout) 100 101 if p.value: self = self.connection = None self.connection.drain_events undefined timeout = None /usr/local/lib/python3.9/site-packages/amqp/connection.py in drain_events(self=, timeout=None) 524 def drain_events(self, timeout=None): 525 # read until message is ready 526 while not self.blocking_read(timeout): 527 pass 528 self = self.blocking_read = > timeout = None /usr/local/lib/python3.9/site-packages/amqp/connection.py in blocking_read(self=, timeout=None) 530 with self.transport.having_timeout(timeout): 531 frame = self.transport.read_frame() 532 return self.on_inbound_frame(frame) 533 534 def on_inbound_method(self, channel_id, method_sig, payload, content): self = self.on_inbound_frame = .on_frame> frame = (1, 0, b'\x00\n\x002\x02\x1d\x0eINTERNAL_ERROR\x00\x00\x00\x00') /usr/local/lib/python3.9/site-packages/amqp/method_framing.py in on_frame(frame=(1, 0, b'\x00\n\x002\x02\x1d\x0eINTERNAL_ERROR\x00\x00\x00\x00')) 51 return False 52 53 callback(channel, method_sig, buf, None) 54 55 elif frame_type == 2: callback = > channel = 0 method_sig = (10, 50) buf = b'\x00\n\x002\x02\x1d\x0eINTERNAL_ERROR\x00\x00\x00\x00' /usr/local/lib/python3.9/site-packages/amqp/connection.py in on_inbound_method(self=, channel_id=0, method_sig=(10, 50), payload=b'\x00\n\x002\x02\x1d\x0eINTERNAL_ERROR\x00\x00\x00\x00', content=None) 536 raise RecoverableConnectionError('Connection already closed') 537 538 return self.channels[channel_id].dispatch_method( 539 method_sig, payload, content, 540 ) self = self.channels = None channel_id = 0 ].dispatch_method undefined method_sig = (10, 50) payload = b'\x00\n\x002\x02\x1d\x0eINTERNAL_ERROR\x00\x00\x00\x00' content = None /usr/local/lib/python3.9/site-packages/amqp/abstract_channel.py in dispatch_method(self=, method_sig=(10, 50), payload=b'\x00\n\x002\x02\x1d\x0eINTERNAL_ERROR\x00\x00\x00\x00', content=None) 154 155 for listener in listeners: 156 listener(*args) 157 158 if one_shot: listener = > args = [541, 'INTERNAL_ERROR', 0, 0] /usr/local/lib/python3.9/site-packages/amqp/connection.py in _on_close(self=, reply_code=541, reply_text='INTERNAL_ERROR', class_id=0, method_id=0) 666 """ 667 self._x_close_ok() 668 raise error_for_code(reply_code, reply_text, 669 (class_id, method_id), ConnectionError) 670 global error_for_code = reply_code = 541 reply_text = 'INTERNAL_ERROR' class_id = 0 method_id = 0 global ConnectionError = InternalError: (0, 0): (541) INTERNAL_ERROR __cause__ = None __class__ = __context__ = None __delattr__ = __dict__ = {'message': 'INTERNAL_ERROR', 'method_name': '', 'method_sig': (0, 0), 'reply_code': 541, 'reply_text': 'INTERNAL_ERROR'} __dir__ = __doc__ = 'AMQP Internal Error.' __eq__ = __format__ = __ge__ = __getattribute__ = __gt__ = __hash__ = __init__ = __init_subclass__ = __le__ = __lt__ = __module__ = 'amqp.exceptions' __ne__ = __new__ = __reduce__ = __reduce_ex__ = __repr__ = __setattr__ = __setstate__ = __sizeof__ = __str__ = __subclasshook__ = __suppress_context__ = False __traceback__ = __weakref__ = None args = (541, 'INTERNAL_ERROR', (0, 0), '') code = 541 message = 'INTERNAL_ERROR' method = (0, 0) method_name = '' method_sig = (0, 0) reply_code = 541 reply_text = 'INTERNAL_ERROR' with_traceback = The above is a description of an error in a Python program. Here is the original traceback: Traceback (most recent call last): File "/usr/local/bin/contrail-svc-monitor", line 8, in sys.exit(server_main()) File "/usr/local/lib/python3.9/site-packages/svc_monitor/svc_monitor.py", line 1011, in server_main main() File "/usr/local/lib/python3.9/site-packages/svc_monitor/svc_monitor.py", line 996, in main vnc_amqp.establish() File "/usr/local/lib/python3.9/site-packages/cfgm_common/vnc_amqp.py", line 35, in establish self._vnc_kombu = VncKombuClient( File "/usr/local/lib/python3.9/site-packages/cfgm_common/vnc_kombu.py", line 368, in __init__ self._start(q_name) File "/usr/local/lib/python3.9/site-packages/cfgm_common/vnc_kombu.py", line 223, in _start self._reconnect_drain(delete_old_q=True) File "/usr/local/lib/python3.9/site-packages/cfgm_common/vnc_kombu.py", line 101, in _reconnect_drain self._consumer = kombu.Consumer(self._channel_drain, File "/usr/local/lib/python3.9/site-packages/kombu/messaging.py", line 414, in __init__ self.revive(self.channel) File "/usr/local/lib/python3.9/site-packages/kombu/messaging.py", line 436, in revive self.declare() File "/usr/local/lib/python3.9/site-packages/kombu/messaging.py", line 450, in declare queue.declare() File "/usr/local/lib/python3.9/site-packages/kombu/entity.py", line 617, in declare self._create_queue(nowait=nowait, channel=channel) File "/usr/local/lib/python3.9/site-packages/kombu/entity.py", line 626, in _create_queue self.queue_declare(nowait=nowait, passive=False, channel=channel) File "/usr/local/lib/python3.9/site-packages/kombu/entity.py", line 655, in queue_declare ret = channel.queue_declare( File "/usr/local/lib/python3.9/site-packages/amqp/channel.py", line 1162, in queue_declare return queue_declare_ok_t(*self.wait( File "/usr/local/lib/python3.9/site-packages/amqp/abstract_channel.py", line 99, in wait self.connection.drain_events(timeout=timeout) File "/usr/local/lib/python3.9/site-packages/amqp/connection.py", line 526, in drain_events while not self.blocking_read(timeout): File "/usr/local/lib/python3.9/site-packages/amqp/connection.py", line 532, in blocking_read return self.on_inbound_frame(frame) File "/usr/local/lib/python3.9/site-packages/amqp/method_framing.py", line 53, in on_frame callback(channel, method_sig, buf, None) File "/usr/local/lib/python3.9/site-packages/amqp/connection.py", line 538, in on_inbound_method return self.channels[channel_id].dispatch_method( File "/usr/local/lib/python3.9/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method listener(*args) File "/usr/local/lib/python3.9/site-packages/amqp/connection.py", line 668, in _on_close raise error_for_code(reply_code, reply_text, amqp.exceptions.InternalError: (0, 0): (541) INTERNAL_ERROR ++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/config-svc-monitor ++ CONTAINER_LOG_DIR=/var/log/contrail/config-svc-monitor ++ mkdir -p /var/log/contrail/config-svc-monitor ++ log_file=/var/log/contrail/config-svc-monitor/console.log ++ touch /var/log/contrail/config-svc-monitor/console.log ++ chmod 600 /var/log/contrail/config-svc-monitor/console.log ++ exec +++ tee -a /var/log/contrail/config-svc-monitor/console.log +++ date ++ echo 'INFO: =================== Mon Mar 2 02:15:15 AM UTC 2026 ===================' INFO: =================== Mon Mar 2 02:15:15 AM UTC 2026 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ ip route get 1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ ip addr show dev ens3 +++ awk '{print $2}' +++ head -n 1 +++ cut -d / -f 1 +++ grep 'inet ' ++ DEFAULT_LOCAL_IP=10.0.0.27 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=256 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 256 ]] ++ [[ -z '' ]] +++ tail -n 1 +++ mount -t hugetlbfs +++ awk '/pagesize=2M/{print($3)}' ++ HUGE_PAGES_2MB_DIR= ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=openstack ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=rbac ++ AUTH_MODE=keystone ++ AUTH_PARAMS= ++ SSL_ENABLE=false ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICS_ALARM_ENABLE=True ++ ANALYTICS_SNMP_ENABLE=True ++ ANALYTICSDB_ENABLE=True ++ ANALYTICS_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICSDB_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICS_SNMP_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:8081 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:8081 ' +++ '[' -n '10.0.0.242:8081 10.0.0.27:8081 10.0.0.23:8081 ' ']' +++ echo '10.0.0.242:8081 10.0.0.27:8081 10.0.0.23:8081' ++ ANALYTICS_SERVERS='10.0.0.242:8081 10.0.0.27:8081 10.0.0.23:8081' +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9042 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9042 ' +++ '[' -n '10.0.0.242:9042 10.0.0.27:9042 10.0.0.23:9042 ' ']' +++ echo '10.0.0.242:9042 10.0.0.27:9042 10.0.0.23:9042' ++ ANALYTICSDB_CQL_SERVERS='10.0.0.242:9042 10.0.0.27:9042 10.0.0.23:9042' ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:8086 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:8086 ' +++ '[' -n '10.0.0.242:8086 10.0.0.27:8086 10.0.0.23:8086 ' ']' +++ echo '10.0.0.242:8086 10.0.0.27:8086 10.0.0.23:8086' ++ COLLECTOR_SERVERS='10.0.0.242:8086 10.0.0.27:8086 10.0.0.23:8086' ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ CONFIGDB_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:8082 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:8082 ' +++ '[' -n '10.0.0.242:8082 10.0.0.27:8082 10.0.0.23:8082 ' ']' +++ echo '10.0.0.242:8082 10.0.0.27:8082 10.0.0.23:8082' ++ CONFIG_SERVERS='10.0.0.242:8082 10.0.0.27:8082 10.0.0.23:8082' +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9161 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9161 ' +++ '[' -n '10.0.0.242:9161 10.0.0.27:9161 10.0.0.23:9161 ' ']' +++ echo '10.0.0.242:9161 10.0.0.27:9161 10.0.0.23:9161' ++ CONFIGDB_SERVERS='10.0.0.242:9161 10.0.0.27:9161 10.0.0.23:9161' +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9041 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9041 ' +++ '[' -n '10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041 ' ']' +++ echo '10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041' ++ CONFIGDB_CQL_SERVERS='10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041' ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=false ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=false ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=false ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.20.0.193,10.20.0.254,10.20.0.14 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.20.0.193,10.20.0.254,10.20.0.14 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+=10.0.0.242:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+=10.0.0.27:2181, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+=10.0.0.23:2181, +++ '[' -n 10.0.0.242:2181,10.0.0.27:2181,10.0.0.23:2181, ']' +++ echo 10.0.0.242:2181,10.0.0.27:2181,10.0.0.23:2181 ++ ZOOKEEPER_SERVERS=10.0.0.242:2181,10.0.0.27:2181,10.0.0.23:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:2181 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:2181 ' +++ '[' -n '10.0.0.242:2181 10.0.0.27:2181 10.0.0.23:2181 ' ']' +++ echo '10.0.0.242:2181 10.0.0.27:2181 10.0.0.23:2181' ++ ZOOKEEPER_SERVERS_SPACE_DELIM='10.0.0.242:2181 10.0.0.27:2181 10.0.0.23:2181' ++ RABBITMQ_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+=10.0.0.242:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+=10.0.0.27:5673, +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+=10.0.0.23:5673, +++ '[' -n 10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673, ']' +++ echo 10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673 ++ RABBITMQ_SERVERS=10.0.0.242:5673,10.0.0.27:5673,10.0.0.23:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=false ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:6379 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:6379 ' +++ '[' -n '10.0.0.242:6379 10.0.0.27:6379 10.0.0.23:6379 ' ']' +++ echo '10.0.0.242:6379 10.0.0.27:6379 10.0.0.23:6379' ++ REDIS_SERVERS='10.0.0.242:6379 10.0.0.27:6379 10.0.0.23:6379' ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=false ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ redis_ssl_config= ++ KAFKA_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.242 +++ local server_address=10.0.0.242 +++ extended_server_list+='10.0.0.242:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.27 +++ local server_address=10.0.0.27 +++ extended_server_list+='10.0.0.27:9092 ' +++ for server in "${server_list[@]}" ++++ echo 10.0.0.23 +++ local server_address=10.0.0.23 +++ extended_server_list+='10.0.0.23:9092 ' +++ '[' -n '10.0.0.242:9092 10.0.0.27:9092 10.0.0.23:9092 ' ']' +++ echo '10.0.0.242:9092 10.0.0.27:9092 10.0.0.23:9092' ++ KAFKA_SERVERS='10.0.0.242:9092 10.0.0.27:9092 10.0.0.23:9092' ++ KAFKA_SSL_ENABLE=false ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=10.0.0.242 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5000 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.242,10.0.0.27,10.0.0.23 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=false ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=false ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=false ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ /v3 == \/\v\2\.\0 ]] ++ [[ openstack == \o\p\e\n\s\t\a\c\k ]] ++ AUTH_MODE=keystone ++ [[ keystone == \k\e\y\s\t\o\n\e ]] ++ AUTH_PARAMS='--admin_password contrail123' ++ AUTH_PARAMS+=' --admin_tenant_name admin' ++ AUTH_PARAMS+=' --admin_user admin' ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ analytics_api_ssl_opts= ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ is_enabled false ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ kafka_ssl_config= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return + wait_for_cassandra ++ echo 10.0.0.242,10.0.0.27,10.0.0.23 ++ cut -d , -f 1 + local cassandra=10.0.0.242 + (( i=1 )) + (( i<=30 )) + echo -e '\035\nquit' + nc -z 10.0.0.242 9041 + echo 'INFO: Cassandra is connected' INFO: Cassandra is connected + sleep 5 + return 0 ++ get_listen_ip_for_node CONFIG +++ find_my_ip_and_order_for_node CONFIG +++ local server_typ=CONFIG_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.242,10.0.0.27,10.0.0.23 +++ cut -d ' ' -f 1 +++ local servers=10.0.0.242,10.0.0.27,10.0.0.23 +++ local server_list= +++ IFS=, +++ read -ra server_list ++++ get_local_ips ++++ cat /proc/net/fib_trie ++++ sort ++++ tr '\n' , ++++ grep -vi host ++++ uniq ++++ awk '/32 host/ { print f } {f=$2}' +++ local local_ips=,10.0.0.27,10.20.0.254,127.0.0.1,172.17.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.242'\''))' +++ local server_ip=10.0.0.242 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.242 ]] +++ [[ ,10.0.0.27,10.20.0.254,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.242, ]] +++ (( ord+=1 )) +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.27'\''))' +++ local server_ip=10.0.0.27 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.27 ]] +++ [[ ,10.0.0.27,10.20.0.254,127.0.0.1,172.17.0.1,, =~ ,10\.0\.0\.27, ]] +++ echo 10.0.0.27 2 +++ return ++ local ip=10.0.0.27 ++ [[ -z 10.0.0.27 ]] ++ echo 10.0.0.27 + host_ip=10.0.0.27 + [[ cql == \c\q\l ]] ++ echo 10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041 ++ sed 's/,/ /g' + cassandra_server_list='10.0.0.242:9041 10.0.0.27:9041 10.0.0.23:9041' + mkdir -p /etc/contrail + cat ++ get_introspect_listen_ip_for_node CONFIG ++ local ip=0.0.0.0 ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ echo 0.0.0.0 + add_ini_params_from_env SVC_MONITOR /etc/contrail/contrail-svc-monitor.conf + local service_name=SVC_MONITOR + local cfg_path=/etc/contrail/contrail-svc-monitor.conf + local delim=__ ++ sed 's/^SVC_MONITOR__//g' ++ cut -d = -f 1 ++ sort ++ set -o posix ++ set ++ grep '^SVC_MONITOR__.*__.*=.*$' + local vars= + local section= + set_third_party_auth_config + [[ keystone != \k\e\y\s\t\o\n\e ]] + local tmp_file=/etc/contrail/contrail-keystone-auth.conf.tmp + cat + [[ /v3 == \/\v\3 ]] + cat + [[ http == \h\t\t\p\s ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n public ]] + echo 'interface = public' + mv -f /etc/contrail/contrail-keystone-auth.conf.tmp /etc/contrail/contrail-keystone-auth.conf + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled false + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + [[ keystone == \k\e\y\s\t\o\n\e ]] + cat + [[ http == \h\t\t\p\s ]] + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + upgrade_old_logs svc-monitor + local template=svc-monitor ++ dirname /var/log/contrail/config-svc-monitor + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/config-svc-monitor + chmod 755 /var/log/contrail/config-svc-monitor + mv -n '/var/log/contrail/*svc-monitor.log*' /var/log/contrail/config-svc-monitor/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/config-svc-monitor + find /var/log/contrail/config-svc-monitor -uid 0 -exec chown 1999:1999 '{}' + + run_service /usr/local/bin/contrail-svc-monitor --conf_file /etc/contrail/contrail-svc-monitor.conf --conf_file /etc/contrail/contrail-keystone-auth.conf + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + mkdir -p /etc/contrail /var/lib/contrail + chown 1999:1999 /etc/contrail /var/lib/contrail + find /etc/contrail -uid 0 -exec chown 1999:1999 '{}' + + chmod 755 /etc/contrail + do_run_service /usr/local/bin/contrail-svc-monitor --conf_file /etc/contrail/contrail-svc-monitor.conf --conf_file /etc/contrail/contrail-keystone-auth.conf + [[ -n 1999 ]] + [[ -n 1999 ]] + mkdir -p /var/crashes + chmod 777 /var/crashes ++ id -un 1999 + local user_name=contrail + export HOME=/home/contrail + HOME=/home/contrail + mkdir -p /home/contrail + chown -R 1999:1999 /home/contrail + exec setpriv --reuid 1999 --regid 1999 --clear-groups --no-new-privs /usr/local/bin/contrail-svc-monitor --conf_file /etc/contrail/contrail-svc-monitor.conf --conf_file /etc/contrail/contrail-keystone-auth.conf 03/02/2026 02:15:24.332 7f4242e0e980 [contrail-svc-monitor] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 03/02/2026 02:15:24.681 7f4242e0e980 [contrail-svc-monitor] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 03/02/2026 02:15:24.682 7f4242e0e980 [contrail-svc-monitor] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/config-svc-monitor/contrail-svc-monitor.log]