kubernetes_monitoring/templates/ConfigMap-GrafanaAgent.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    {{ toYaml $.Values.nfc_monitoring.grafana_agent.labels | nindent 4 }}
    app.kubernetes.io/managed-by: {{ $.Release.Service }}
    app.kubernetes.io/part-of: {{ $.Chart.Name }}
    app.kubernetes.io/version: {{ $.Chart.Version }}
  name: grafana-agent
  namespace: monitoring
data:
  agent.yaml: |
    metrics:
      wal_directory: /tmp/wal



    logs:
      positions_directory: "/tmp"

      configs:

      - name: node-logs
        clients:
          - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:{{ .Values.nfc_monitoring.loki.service_port }}/loki/api/v1/push
            backoff_config:
              min_period: 10s
              max_period: 5m
              max_retries: 10

        limits_config:
          readline_rate_enabled: true
          readline_rate: 250
          readline_burst: 750
          readline_rate_drop: false
          max_streams: 500

        scrape_configs:

          - job_name: systemd-journal
            journal:
              path: /host/root/run/log/journal
              json: true
              max_age: 24h
            relabel_configs:
                
            - source_labels:
              - __journal__systemd_unit
              target_label: systemd_unit
            - source_labels:
              - __journal__hostname
              target_label: node
            - source_labels:
              - __journal_syslog_identifier
              target_label: syslog_identifier
            - target_label: "job"
              replacement: "systemd-journal"
            - target_label: "job_name"
              replacement: "journal-logs"
            - target_label: hostname
              replacement: "${HOSTNAME}"

            pipeline_stages:
              - json:
                  expressions:
                    pid: _PID
                    userId: _UID
                    application: _COMM
                    priority: PRIORITY

              - labels:
                  application:
                  #level: 
                  pid: 
                  userId:
                  priority:
              - template:
                  source: level
                  template: '{{"{{"}} ToLower .Value {{"}}"}}'
              - match:
                  selector: '{priority="7"}'
                  stages:
                  - template:
                      source: level
                      template: 'debug'
              - match:
                  selector: '{priority="6"}'
                  stages:
                  - template:
                      source: level
                      template: 'info'
              - match:
                  selector: '{priority="5"}'
                  stages:
                  - template:
                      source: level
                      template: 'notice'
              - match:
                  selector: '{priority="4"}'
                  stages:
                  - template:
                      source: level
                      template: 'warning'
              - match:
                  selector: '{priority="3"}'
                  stages:
                  - template:
                      source: level
                      template: 'error'
              - match:
                  selector: '{priority="2"}'
                  stages:
                  - template:
                      source: level
                      template: 'crit'
              - match:
                  selector: '{priority="1"}'
                  stages:
                  - template:
                      source: level
                      template: 'alert'
              - match:
                  selector: '{priority="0"}'
                  stages:
                  - template:
                      source: level
                      template: 'emerg'
              - labels:
                  level: 


          - job_name: var-logs
            static_configs:
              - targets: [localhost]
                labels:
                  job_name: 'aptitude'
                  __path__: /var/log/apt/*.log
              - targets: [localhost]
                labels:
                  job_name: 'dpkg'
                  __path__: /var/log/dpkg.log
              - targets: [localhost]
                labels:
                  job_name: 'messages'
                  __path__: /var/log/messages
              - targets: [localhost]
                labels:
                  job_name: 'syslog'
                  __path__: /var/log/syslog
              - targets: [localhost]
                labels:
                  job_name: 'kernlog'
                  __path__: /var/log/kern.log
              - targets: [localhost]
                labels:
                  job_name: 'auth-log'
                  __path__: /var/log/auth.log
              - targets: [localhost]
                labels:
                  job_name: 'boot-log'
                  __path__: /var/log/boot.log
              - targets: [localhost]
                labels:
                  job_name: 'daemon-log'
                  __path__: /var/log/daemon.log
              - targets: [localhost]
                labels:
                  job_name: 'faillog-log'
                  __path__: /var/log/faillog
              - targets: [localhost]
                labels:
                  job_name: 'lastlog-log'
                  __path__: /var/log/lastlog

            relabel_configs:
              - target_label: hostname
                replacement: "${HOSTNAME}"
              - target_label: job
                replacement: log-files
              - target_label: node
                replacement: "${HOSTNAME}"

            pipeline_stages:
              - timestamp:
                  format: RFC3339
                  source: time



          - job_name: pod-logs
            kubernetes_sd_configs:
              - role: pod

            pipeline_stages:
              - cri: {}

              - regex:
                  expression: '(\s|\t|level=)?(?P<level>trace|debug|info|warn|error|TRACE|DEBUG|INFO|WARN|ERROR)(\s|\t)'

              - labels:
                  level:

            relabel_configs:
              - source_labels:
                  - __meta_kubernetes_pod_node_name
                target_label: __host__
              - action: labelmap
                regex: __meta_kubernetes_pod_label_(.+)
              - action: replace
                source_labels:
                  - __meta_kubernetes_namespace
                target_label: namespace
              - target_label: job
                replacement: kubernetes_sd
              - action: replace
                source_labels:
                  - __meta_kubernetes_pod_name
                target_label: pod
              - action: replace
                source_labels:
                  - __meta_kubernetes_pod_container_name
                target_label: container
              - replacement: /var/log/pods/*$1/*.log
                separator: /
                source_labels:
                  - __meta_kubernetes_pod_uid
                  - __meta_kubernetes_pod_container_name
                target_label: __path__
              - target_label: "job_name"
                replacement: "kubernetes-logs"
              - target_label: hostname
                replacement: "${HOSTNAME}"
              - target_label: node
                source_labels:
                  - __meta_kubernetes_pod_node_name


    integrations:

      node_exporter:
        enabled: true
        rootfs_path: /host/root
        sysfs_path: /host/sys
        procfs_path: /host/proc
        udev_data_path: /host/root/run/udev/data

        # collector.filesystem.ignored-mount-points: ^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+|/run/containerd/io.containerd.+)($|/)
        filesystem_mount_points_exclude: "^/(dev|proc|sys|var/lib/docker/.+|/run/containerd/io.containerd.+)($|/)"
        #filesystem_fs_types_exclude: "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|shm|squashfs|sysfs|tracefs)$"
        filesystem_fs_types_exclude: "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|ugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|ocfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"


        netclass_ignored_devices: "^(veth.*|cali.*|[a-f0-9]{15})$"
        netdev_device_exclude: "^(veth.*|cali.*|[a-f0-9]{15})$"


        include_exporter_metrics: true
        enable_collectors:
          - uname


  syslog_server.yaml: |
    # REF: https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#example-syslog-config
    server:
      http_listen_port: 9080
      grpc_listen_port: 0

    positions:
      filename: /tmp/syslog_server_positions.yaml

    clients:
      - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:{{ .Values.nfc_monitoring.loki.service_port }}/loki/api/v1/push

    scrape_configs:
      - job_name: syslog
        syslog:
          listen_address: 0.0.0.0:1514
          labels:
            job: "syslog"
        relabel_configs:
          - source_labels: ['__syslog_message_hostname']
            target_label: 'host'