kubernetes_monitoring/templates/ConfigMap-GrafanaAgent.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    app.kubernetes.io/instance: k8s
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: grafana-agent
    app.kubernetes.io/managed-by: {{ $.Release.Service }}
    app.kubernetes.io/part-of: {{ $.Chart.Name }}
    app.kubernetes.io/version: {{ $.Chart.Version }}
  name: grafana-agent
  namespace: monitoring
data:
  agent.yaml: |
    metrics:
      wal_directory: /tmp/wal

    logs:
      # Choose a directory to save the last read position of log files at.
      # This directory will be created if it doesn't already exist.
      positions_directory: "/tmp"
    
      configs:

      - name: journal
        clients:
          - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:3100/loki/api/v1/push
        scrape_configs:
        
        - job_name: systemd-journal
          journal:
            labels:
              job: node-journal
            path: /host/root/run/log/journal
            json: true
          relabel_configs:
              
          - source_labels:
            - __journal__systemd_unit
            target_label: systemd_unit
          - source_labels:
            - __journal__hostname
            target_label: node
          - source_labels:
            - __journal_syslog_identifier
            target_label: syslog_identifier
          - target_label: "job_name"
            replacement: "journal"

          pipeline_stages:
            - json:
                expressions:
                  pid: _PID
                  userId: _UID
                  application: _COMM
                  priority: PRIORITY

            - labels:
                application:
                #level: 
                pid: 
                userId:
                priority:
            - template:
                source: level
                template: '{{"{{"}} ToLower .Value {{"}}"}}'
            - match:
                selector: '{priority="7"}'
                stages:
                - template:
                    source: level
                    template: 'debug'
            - match:
                selector: '{priority="6"}'
                stages:
                - template:
                    source: level
                    template: 'info'
            - match:
                selector: '{priority="5"}'
                stages:
                - template:
                    source: level
                    template: 'notice'
            - match:
                selector: '{priority="4"}'
                stages:
                - template:
                    source: level
                    template: 'warning'
            - match:
                selector: '{priority="3"}'
                stages:
                - template:
                    source: level
                    template: 'error'
            - match:
                selector: '{priority="2"}'
                stages:
                - template:
                    source: level
                    template: 'crit'
            - match:
                selector: '{priority="1"}'
                stages:
                - template:
                    source: level
                    template: 'alert'
            - match:
                selector: '{priority="0"}'
                stages:
                - template:
                    source: level
                    template: 'emerg'
            - labels:
                level: 

        # - job_name: varlogs
        #   static_configs:
        #     - targets: [localhost]
        #       labels:
        #         # cluster: dev
        #         job: 'container logs'
        #         __path__: /var/log/pods/*/*/*.log
        #   pipeline_stages:
        #     - json:
        #         expressions:
        #           namespace: namespace
        #           level: level
        #     - regex:
        #         source: filename
        #         # expression: '/var/log/pods/(.+)/(?P<pod>.+)/(*.log)'
        #         # expression: '/var/log/pods/.+/(?P<pod>\\S+?)'
        #         #expression: '/var/log/pods/.+/(?P<pod>\\S+?)/*.log'
        #         #expression: '/var/log/pods/.+/(?P<pod>\S+?)/*.log'
        #         expression: '/var/log/pods/.+/(?P<container>\S+?)/.+log'
        #     - regex:
        #         source: filename
        #         expression: '/var\/log\/pods\/.*_(?P<pod>\S+?)_.*\/.+\/.+log'
        #     - regex:
        #         source: filename
        #         expression: '/var/log/pods/(?P<namespace>\S+?)_.*/.+/.+log'
        #     - regex:
        #         #source: message
        #         expression: '\s?level=(?P<level>\S+)\s+'
        #     - regex:
        #         #source: message
        #         expression: '\s?(?P<level>debug|info|warn|error)\s'
        #     - template:
        #         source: level
        #         template: '{{"{{"}} ToLower .Value {{"}}"}}'
        #     - labels:
        #         container:
        #         level:
        #         namespace:
        #         pod:
          #relabel_configs:
            
            # - source_labels: [__filename__]
            #   separator: '/'
            #   regex: '/var/log/pods/*/(.*)/*.log'
            #   replacement: '${1}'
            #   target_label: pod
            # - source_labels: [__filename__]
            #   replacement: '/var/log/pods/*/${1}/*.log'
            #   target_label: pod
            # - replacement: /var/log/pods/*/${1}/*.log
            #   separator: /
            #   source_labels:
            #     - __meta_kubernetes_pod_uid
            #     - __meta_kubernetes_pod_container_name
            #   target_label: pod



      #       - targets: [localhost]
      #         labels:
      #           job: varlogs
      #           __path__: /var/logs/*
      #       - targets: [localhost]
      #         labels:
      #           job: varlogs
      #           __path__: /host/root/var/log/*
      #       - targets: [localhost]
      #         labels:
      #           job: varlogs
      #           __path__: /host/root/var/logs/*
      #       - targets: [localhost]
      #         labels:
      #           job: varlogs
      #           __path__: /host/root/var/log/containers/*
      #       - targets: [localhost]
      #         labels:
      #           job: varlogs
      #           __path__: /host/root/var/log/*/*






      - name: kubernetes
        clients:
        - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:3100/loki/api/v1/push
          # basic_auth:
          #   username: YOUR_LOKI_USERNAME
          #   password: YOUR_LOKI_PASSWORD
          external_labels:
            #cluster: dev-cluster
        positions:
          filename: /tmp/kub-positions.yaml
        target_config:
          sync_period: 10s
        scrape_configs:
        - job_name: pod-logs

          kubernetes_sd_configs:
            - role: pod

          pipeline_stages:
            - cri: {}

            - regex:
                #source: msg
                expression: '(\s|\t|level=)?(?P<level>trace|debug|info|warn|error|TRACE|DEBUG|INFO|WARN|ERROR)(\s|\t)'

            - labels:
                level:

          relabel_configs:
            - source_labels:
                - __meta_kubernetes_pod_node_name
              target_label: __host__
            - action: labelmap
              regex: __meta_kubernetes_pod_label_(.+)
            - action: replace
              source_labels:
                - __meta_kubernetes_namespace
              target_label: namespace
            - target_label: job
              # source_labels:
              #     - __meta_kubernetes_namespace
              replacement: loki/ingester_grafana-agent
            - action: replace
              source_labels:
                - __meta_kubernetes_pod_name
              target_label: pod
            - action: replace
              source_labels:
                - __meta_kubernetes_pod_container_name
              target_label: container
            - replacement: /var/log/pods/*$1/*.log
              separator: /
              source_labels:
                - __meta_kubernetes_pod_uid
                - __meta_kubernetes_pod_container_name
              target_label: __path__
            - target_label: "job_name"
              replacement: "kubernetes_sd"



    integrations:

      node_exporter:
        enabled: true
        rootfs_path: /host/root
        sysfs_path: /host/sys
        procfs_path: /host/proc
        udev_data_path: /host/root/run/udev/data

        # collector.filesystem.ignored-mount-points: ^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+|/run/containerd/io.containerd.+)($|/)
        filesystem_mount_points_exclude: "^/(dev|proc|sys|var/lib/docker/.+|/run/containerd/io.containerd.+)($|/)"
        filesystem_fs_types_exclude: "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|shm|squashfs|sysfs|tracefs)$"

        scrape_integration: true

        include_exporter_metrics: true
        enable_collectors:
          - uname