# Inputs
# Included from conf/input/00-global.conf.j2:
#jinja2: trim_blocks: False
<source>
  @type tail
  path /var/log/kolla/barbican/*.log,/var/log/kolla/glance/*.log,/var/log/kolla/heat/*.log,/var/log/kolla/horizon/*.log,/var/log/kolla/keystone/*.log,/var/log/kolla/neutron/*.log,/var/log/kolla/nova/*.log
  exclude_path ["/var/log/kolla/monasca/agent*.log",
                "/var/log/kolla/monasca/monasca-api.log",
                "/var/log/kolla/neutron/dnsmasq.log",
                "/var/log/kolla/ironic/dnsmasq.log",
                "/var/log/kolla/*/*-access.log",
                "/var/log/kolla/*/*-error.log",
                "/var/log/kolla/*/*_access.log",
                "/var/log/kolla/*/*_error.log"]
  pos_file /var/run/td-agent/kolla-openstack.pos
  tag kolla.*
  ignore_repeated_permission_error true
  enable_watch_timer false
  <parse>
    @type multiline
    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
    format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
    time_key Timestamp
    keep_time_key true
    time_format %F %T.%L
  </parse>
</source>
# Included from conf/input/01-syslog.conf.j2:
<source>
  @type syslog
  port 5140
  bind 10.0.0.17
  tag syslog
  severity_key log_level
  source_hostname_key Hostname
  <parse>
    @type regexp
    expression /^(?<Payload>.*)$/
  </parse>
</source>
# Included from conf/input/02-mariadb.conf.j2:
# mysqld and its wrapper script mysqld_safe output logs using a different timestamp.
# Defer parsing the logs until the different formats have been retagged.
<source>
  @type tail
  path /var/log/kolla/mariadb/mariadb.log
  pos_file /var/run/td-agent/mariadb.pos
  tag infra.mariadb
  enable_watch_timer false
  <parse>
    @type multiline
    format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) /
    format1 /^(?<Payload>.*)/
  </parse>
</source>
<source>
  @type tail
  path /var/log/kolla/mariadb/xinetd.log
  pos_file /var/run/td-agent/mariadb-xinetd.pos
  tag infra.mariadb-xinetd
  ignore_repeated_permission_error true
  enable_watch_timer false
  <parse>
    @type multiline
    format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: /
    format1 /^(?<Timestamp>\S+) (?<Payload>.*)?$/
    time_key Timestamp
    keep_time_key true
    time_format %y/%m/%d@%T
  </parse>
</source>
# Included from conf/input/03-rabbitmq.conf.j2:
<source>
  @type tail
  path /var/log/kolla/rabbitmq/rabbit@cn-jenkins-deploy-platform-ansible-os-2155-1.log
  pos_file /var/run/td-agent/rabbit.pos
  tag infra.rabbit
  enable_watch_timer false
  <parse>
    @type multiline
    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/
    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?<log_level>\w+)\] (?<Payload>.*)/
  </parse>
</source>
# Included from conf/input/04-openstack-wsgi.conf.j2:
# Note (blallau): to manage Apache and WSGI log files
<source>
  @type tail
  path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log,/var/log/kolla/*/*_access.log,/var/log/kolla/*/*_error.log
  pos_file /var/run/td-agent/kolla-openstack-wsgi.pos
  tag kolla.*
  enable_watch_timer false
  <parse>
    @type regexp
    expression /^(?<Payload>.*)$/
  </parse>
</source>
# Included from conf/input/05-libvirt.conf.j2:
<source>
  @type tail
  path /var/log/kolla/libvirt/libvirtd.log
  pos_file /var/run/td-agent/libvirt.pos
  tag infra.libvirt
  enable_watch_timer false
  <parse>
    @type regexp
    expression /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
    time_key Timestamp
    time_format %F %T.%L%z
  </parse>
</source>
# Included from conf/input/06-zookeeper.conf.j2:
<source>
  @type tail
  path /var/log/kolla/zookeeper/zookeeper.log
  pos_file /var/run/td-agent/zookeeper.pos
  tag infra.*
  <parse>
    @type multiline
    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) \[(?<server_id>\S+)\] \S+ (?<log_level>\S+) (?<Payload>.*)$/
    time_key Timestamp
  </parse>
</source>
# Included from conf/input/07-kafka.conf.j2:
<source>
  @type tail
  path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
  pos_file /var/run/td-agent/kafka.pos
  tag infra.*
  <parse>
    @type multiline
    format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/
    format1 /^\[(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] (?<log_level>\S+) (?<Payload>.*)$/
    time_key Timestamp
  </parse>
</source>
# Included from conf/input/09-monasca.conf.j2:
<source>
  @type tail
  path /var/log/kolla/monasca/agent*.log
  pos_file /var/run/td-agent/monasca-agent.pos
  tag kolla.*
  <parse>
    @type multiline
    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+) \| (?<log_level>\S+) \| (?<programname>\S+) \| (?<Payload>.*)$/
    time_key Timestamp
  </parse>
</source>
# Included from conf/input/10-openvswitch.conf.j2:
<source>
  @type tail
  path /var/log/kolla/openvswitch/ovs-vswitchd.log
  pos_file /var/run/td-agent/openvswitch.pos
  tag infra.openvswitch
  enable_watch_timer false
  <parse>
    @type multiline
    format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/
    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?<module>\S+)\|(?<log_level>\S+)\|(?<Payload>.*)/
    time_key Timestamp
    time_format %FT%T.%L
  </parse>
</source>

<source>
  @type tail
  path /var/log/kolla/openvswitch/ovsdb-server.log
  pos_file /var/run/td-agent/openvswitchdb.pos
  tag infra.openvswitchdb
  enable_watch_timer false
  <parse>
    @type multiline
    format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/
    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?<module>\S+)\|(?<log_level>\S+)\|(?<Payload>.*)/
    time_key Timestamp
    time_format %FT%T.%L
  </parse>
</source>

# Filters
# Included from conf/filter/00-record_transformer.conf.j2:
<filter *.var.log.kolla.*.*.log>
    @type record_transformer
    <record>
        Hostname ${hostname}
        Logger openstack.${tag_parts[4]}
        programname ${tag_parts[5]}
    </record>
</filter>

<filter infra.var.log.kolla.*.*.log>
    @type record_transformer
    <record>
        Logger ${tag_parts[4]}
    </record>
</filter>

<filter infra.*>
    @type record_transformer
    <record>
        Hostname ${hostname}
        programname ${tag_parts[1]}
    </record>
</filter>



# Rename internal Fluent message field to match other logs. This removes
# all other fields by default, including the original message field. This is
# intented to avoid duplication of the log message and to prevent passing
# invalid dimensions to Monasca, if it is enabled. Note that if this step
# is moved to the format folder, then it will applied after the second step
# below which will break the logic.
<filter fluent.**>
    @type parser
    key_name message
    format /^(?<Payload>.*)$/
</filter>

<filter fluent.**>
    @type record_transformer
    <record>
        Hostname "#{Socket.gethostname}"
        programname ${tag_parts[0]}
        log_level ${tag_parts[1]}
    </record>
</filter>


# Included from conf/filter/01-rewrite.conf.j2:
<match kolla.var.log.kolla.*.*.log>
    @type rewrite_tag_filter
    capitalize_regex_backreference yes
  <rule>
    key     programname
    pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|octavia-api-access|placement-api-access)$
    tag apache_access
  </rule>
  <rule>
    key     programname
    pattern ^(aodh_wsgi_access|barbican_api_uwsgi_access|zun_api_wsgi_access|vitrage_wsgi_access)$
    tag wsgi_access
  </rule>
  <rule>
    key     programname
    pattern ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(sahara-api|sahara-engine)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(neutron-server|neutron-openvswitch-agent|neutron-ns-metadata-proxy|neutron-metadata-agent|neutron-l3-agent|neutron-dhcp-agent)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(magnum-conductor|magnum-api)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(keystone)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(heat-engine|heat-api|heat-api-cfn)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(glance-api)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(cloudkitty-storage-init|cloudkitty-processor|cloudkitty-dbsync|cloudkitty-api)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(ceilometer-polling|ceilometer-agent-notification)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(barbican-api|barbican-worker|barbican-keystone-listener|barbican-db-manage|app)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(aodh-notifier|aodh-listener|aodh-evaluator|aodh-dbsync)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(cyborg-api|cyborg-conductor|cyborg-agent)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(cinder-api|cinder-scheduler|cinder-manage|cinder-volume|cinder-backup|privsep-helper)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(mistral-server|mistral-engine|mistral-executor)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(designate-api|designate-central|designate-manage|designate-mdns|designate-sink|designate-worker)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(manila-api|manila-data|manila-manage|manila-share|manila-scheduler)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(trove-api|trove-conductor|trove-manage|trove-taskmanager)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(murano-api|murano-engine)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(senlin-api|senlin-conductor|senlin-engine|senlin-health-manager)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(watcher-api|watcher-applier|watcher-db-manage|watcher-decision-engine)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(freezer-api|freezer-api_access|freezer-manage)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(octavia-api|octavia-health-manager|octavia-housekeeping|octavia-worker)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(zun-api|zun-compute|zun-cni-daemon)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(kuryr-server)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(gnocchi-api|gnocchi-statsd|gnocchi-metricd|gnocchi-upgrade)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(ironic-api|ironic-conductor|ironic-inspector)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(tacker-server|tacker-conductor)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(vitrage-ml|vitrage-notifier|vitrage-graph|vitrage-persistor)$
    tag openstack_python
</rule>
  <rule>
    key     programname
    pattern ^(blazar-api|blazar-manager)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(monasca-api|monasca-notification|monasca-persister|agent-collector|agent-forwarder|agent-statsd)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern ^(masakari-engine|masakari-api)$
    tag openstack_python
  </rule>
  <rule>
    key     programname
    pattern .+
    tag unmatched
  </rule>
</match>

# Retag log messages from MariaDB according to log format
<match infra.mariadb>
  @type rewrite_tag_filter
  <rule>
    key Payload
    pattern /^\d{6}/
    tag infra.mariadb.mysqld_safe
  </rule>
  <rule>
    key Payload
    pattern /^\d{4}-\d{2}-\d{2}/
    tag infra.mariadb.mysqld
  </rule>
</match>
# Included from conf/filter/02-parser.conf.j2:
# Parse MariaDB logs with 6 digit date format (mysqld_safe)
<filter infra.mariadb.mysqld_safe>
    @type parser
    format /^(?<Timestamp>\d{6} {1,2}\d{1,2}:\d{1,2}:\d{1,2}) +(?<Payload>mysqld_safe .*)/
    time_format %y%m%d %k:%M:%S
    time_key Timestamp
    key_name Payload
    reserve_data true
</filter>

# Parse MariaDB logs with 8 digit date format (mysqld)
<filter infra.mariadb.mysqld>
    @type parser
    format /^(?<Timestamp>\d{4}-\d{2}-\d{2} {1,2}\d{1,2}:\d{1,2}:\d{1,2}) +(?<Payload>\w+ +(\[(?<log_level>\w+)\]|\w+: +(?<log_level>\w+):).*)/
    time_format %Y-%m-%d %k:%M:%S
    time_key Timestamp
    key_name Payload
    reserve_data true
</filter>

# Re-add timestamp record now that the log date has been parsed
<filter infra.mariadb.*>
  @type record_transformer
  <record>
    timestamp ${time}
  </record>
</filter>

# Formats
# Included from conf/format/apache_access.conf.j2:
<filter apache_access>
  @type parser
  reserve_data true
  key_name Payload
  <parse>
    @type grok
    grok_pattern \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:%{NUMBER:http_bytes}|-) (?:%{NUMBER:http_response_time_us}|-) "%{DATA:referrer}" "%{DATA:agent}"
    time_key Timestamp
    time_format %d/%b/%Y:%H:%M:%S %z
    keep_time_key true
  </parse>
</filter>
# Included from conf/format/wsgi_access.conf.j2:
<filter wsgi_access>
  @type parser
  reserve_data true
  key_name Payload
  <parse>
    @type grok
    grok_pattern %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:%{NUMBER:http_bytes}|-) (?:%{NUMBER:http_response_time_us}|-) %{QS:referrer} %{QS:agent}
    time_key Timestamp
    time_format %d/%b/%Y:%H:%M:%S %z
    keep_time_key true
  </parse>
</filter>

# Outputs
# Included from conf/output/00-local.conf.j2: