From 817b83865555cbfa49dad03b2beffa2a93c45124 Mon Sep 17 00:00:00 2001 From: Jon Date: Tue, 26 Sep 2023 01:30:06 +0930 Subject: [PATCH] refactor(grafana_agent): clean up config file !1 --- templates/ConfigMap-GrafanaAgent.yaml | 497 +++++++++++--------------- 1 file changed, 203 insertions(+), 294 deletions(-) diff --git a/templates/ConfigMap-GrafanaAgent.yaml b/templates/ConfigMap-GrafanaAgent.yaml index 3582f43..623ea72 100644 --- a/templates/ConfigMap-GrafanaAgent.yaml +++ b/templates/ConfigMap-GrafanaAgent.yaml @@ -17,314 +17,223 @@ data: wal_directory: /tmp/wal logs: - # Choose a directory to save the last read position of log files at. - # This directory will be created if it doesn't already exist. positions_directory: "/tmp" - + configs: - - name: journal + - name: node-logs clients: - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:{{ .Values.nfc_monitoring.loki.service_port }}/loki/api/v1/push - scrape_configs: - - - job_name: systemd-journal - journal: - labels: - job: node-journal - path: /host/root/run/log/journal - json: true - relabel_configs: - - - source_labels: - - __journal__systemd_unit - target_label: systemd_unit - - source_labels: - - __journal__hostname - target_label: node - - source_labels: - - __journal_syslog_identifier - target_label: syslog_identifier - - target_label: "job_name" - replacement: "journal" - - target_label: hostname - replacement: "${HOSTNAME}" + backoff_config: + min_period: 10s + max_period: 5m + max_retries: 10 - pipeline_stages: - - json: - expressions: - pid: _PID - userId: _UID - application: _COMM - priority: PRIORITY + limits_config: + readline_rate_enabled: true + readline_rate: 250 + readline_burst: 750 + readline_rate_drop: false + max_streams: 500 - - labels: - application: - #level: - pid: - userId: - priority: - - template: - source: level - template: '{{"{{"}} ToLower .Value {{"}}"}}' - - match: - selector: '{priority="7"}' - stages: - - template: - source: level - template: 'debug' - - match: - selector: '{priority="6"}' - stages: - - template: - source: level - template: 'info' - - match: - selector: '{priority="5"}' - stages: - - template: - source: level - template: 'notice' - - match: - selector: '{priority="4"}' - stages: - - template: - source: level - template: 'warning' - - match: - selector: '{priority="3"}' - stages: - - template: - source: level - template: 'error' - - match: - selector: '{priority="2"}' - stages: - - template: - source: level - template: 'crit' - - match: - selector: '{priority="1"}' - stages: - - template: - source: level - template: 'alert' - - match: - selector: '{priority="0"}' - stages: - - template: - source: level - template: 'emerg' - - labels: - level: - - - - name: log-files - clients: - - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:{{ .Values.nfc_monitoring.loki.service_port }}/loki/api/v1/push - positions: - filename: /tmp/var-logfile.yaml - target_config: - sync_period: 10s scrape_configs: - - job_name: var-logs - static_configs: - - targets: [localhost] - labels: - job_name: 'aptitude' - __path__: /var/log/apt/*.log - - targets: [localhost] - labels: - job_name: 'dpkg' - __path__: /var/log/dpkg.log - - targets: [localhost] - labels: - job_name: 'messages' - __path__: /var/log/messages - - targets: [localhost] - labels: - job_name: 'syslog' - __path__: /var/log/syslog - - targets: [localhost] - labels: - job_name: 'kernlog' - __path__: /var/log/kern.log - - targets: [localhost] - labels: - job_name: 'auth-log' - __path__: /var/log/auth.log - - targets: [localhost] - labels: - job_name: 'boot-log' - __path__: /var/log/boot.log - - targets: [localhost] - labels: - job_name: 'daemon-log' - __path__: /var/log/daemon.log - - targets: [localhost] - labels: - job_name: 'faillog-log' - __path__: /var/log/faillog - - targets: [localhost] - labels: - job_name: 'lastlog-log' - __path__: /var/log/lastlog - - relabel_configs: - - target_label: hostname - replacement: "${HOSTNAME}" - - target_label: job - replacement: log-files - - - # - job_name: varlogs - # static_configs: - # - targets: [localhost] - # labels: - # # cluster: dev - # job: 'container logs' - # __path__: /var/log/pods/*/*/*.log - # pipeline_stages: - # - json: - # expressions: - # namespace: namespace - # level: level - # - regex: - # source: filename - # # expression: '/var/log/pods/(.+)/(?P.+)/(*.log)' - # # expression: '/var/log/pods/.+/(?P\\S+?)' - # #expression: '/var/log/pods/.+/(?P\\S+?)/*.log' - # #expression: '/var/log/pods/.+/(?P\S+?)/*.log' - # expression: '/var/log/pods/.+/(?P\S+?)/.+log' - # - regex: - # source: filename - # expression: '/var\/log\/pods\/.*_(?P\S+?)_.*\/.+\/.+log' - # - regex: - # source: filename - # expression: '/var/log/pods/(?P\S+?)_.*/.+/.+log' - # - regex: - # #source: message - # expression: '\s?level=(?P\S+)\s+' - # - regex: - # #source: message - # expression: '\s?(?Pdebug|info|warn|error)\s' - # - template: - # source: level - # template: '{{"{{"}} ToLower .Value {{"}}"}}' - # - labels: - # container: - # level: - # namespace: - # pod: - #relabel_configs: - - # - source_labels: [__filename__] - # separator: '/' - # regex: '/var/log/pods/*/(.*)/*.log' - # replacement: '${1}' - # target_label: pod - # - source_labels: [__filename__] - # replacement: '/var/log/pods/*/${1}/*.log' - # target_label: pod - # - replacement: /var/log/pods/*/${1}/*.log - # separator: / - # source_labels: - # - __meta_kubernetes_pod_uid - # - __meta_kubernetes_pod_container_name - # target_label: pod - - - - # - targets: [localhost] - # labels: - # job: varlogs - # __path__: /var/logs/* - # - targets: [localhost] - # labels: - # job: varlogs - # __path__: /host/root/var/log/* - # - targets: [localhost] - # labels: - # job: varlogs - # __path__: /host/root/var/logs/* - # - targets: [localhost] - # labels: - # job: varlogs - # __path__: /host/root/var/log/containers/* - # - targets: [localhost] - # labels: - # job: varlogs - # __path__: /host/root/var/log/*/* - - - - - - - - name: kubernetes - clients: - - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:{{ .Values.nfc_monitoring.loki.service_port }}/loki/api/v1/push - # basic_auth: - # username: YOUR_LOKI_USERNAME - # password: YOUR_LOKI_PASSWORD - external_labels: - #cluster: dev-cluster - positions: - filename: /tmp/kub-positions.yaml - target_config: - sync_period: 10s - scrape_configs: - - job_name: pod-logs - - kubernetes_sd_configs: - - role: pod - - pipeline_stages: - - cri: {} - - - regex: - #source: msg - expression: '(\s|\t|level=)?(?Ptrace|debug|info|warn|error|TRACE|DEBUG|INFO|WARN|ERROR)(\s|\t)' - - - labels: - level: - - relabel_configs: + - job_name: systemd-journal + journal: + path: /host/root/run/log/journal + json: true + max_age: 24h + relabel_configs: + - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - # - target_label: job - # # source_labels: - # # - __meta_kubernetes_namespace - # replacement: loki/ingester_grafana-agent - - target_label: job - replacement: grafana-agent - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ + - __journal__systemd_unit + target_label: systemd_unit + - source_labels: + - __journal__hostname + target_label: node + - source_labels: + - __journal_syslog_identifier + target_label: syslog_identifier + - target_label: "job" + replacement: "systemd-journal" - target_label: "job_name" - replacement: "kubernetes_sd" + replacement: "journal-logs" - target_label: hostname replacement: "${HOSTNAME}" + pipeline_stages: + - json: + expressions: + pid: _PID + userId: _UID + application: _COMM + priority: PRIORITY + + - labels: + application: + #level: + pid: + userId: + priority: + - template: + source: level + template: '{{"{{"}} ToLower .Value {{"}}"}}' + - match: + selector: '{priority="7"}' + stages: + - template: + source: level + template: 'debug' + - match: + selector: '{priority="6"}' + stages: + - template: + source: level + template: 'info' + - match: + selector: '{priority="5"}' + stages: + - template: + source: level + template: 'notice' + - match: + selector: '{priority="4"}' + stages: + - template: + source: level + template: 'warning' + - match: + selector: '{priority="3"}' + stages: + - template: + source: level + template: 'error' + - match: + selector: '{priority="2"}' + stages: + - template: + source: level + template: 'crit' + - match: + selector: '{priority="1"}' + stages: + - template: + source: level + template: 'alert' + - match: + selector: '{priority="0"}' + stages: + - template: + source: level + template: 'emerg' + - labels: + level: + + + - job_name: var-logs + static_configs: + - targets: [localhost] + labels: + job_name: 'aptitude' + __path__: /var/log/apt/*.log + - targets: [localhost] + labels: + job_name: 'dpkg' + __path__: /var/log/dpkg.log + - targets: [localhost] + labels: + job_name: 'messages' + __path__: /var/log/messages + - targets: [localhost] + labels: + job_name: 'syslog' + __path__: /var/log/syslog + - targets: [localhost] + labels: + job_name: 'kernlog' + __path__: /var/log/kern.log + - targets: [localhost] + labels: + job_name: 'auth-log' + __path__: /var/log/auth.log + - targets: [localhost] + labels: + job_name: 'boot-log' + __path__: /var/log/boot.log + - targets: [localhost] + labels: + job_name: 'daemon-log' + __path__: /var/log/daemon.log + - targets: [localhost] + labels: + job_name: 'faillog-log' + __path__: /var/log/faillog + - targets: [localhost] + labels: + job_name: 'lastlog-log' + __path__: /var/log/lastlog + + relabel_configs: + - target_label: hostname + replacement: "${HOSTNAME}" + - target_label: job + replacement: log-files + - target_label: node + replacement: "${HOSTNAME}" + + pipeline_stages: + - timestamp: + format: RFC3339 + source: time + + + + - job_name: pod-logs + kubernetes_sd_configs: + - role: pod + + pipeline_stages: + - cri: {} + + - regex: + expression: '(\s|\t|level=)?(?Ptrace|debug|info|warn|error|TRACE|DEBUG|INFO|WARN|ERROR)(\s|\t)' + + - labels: + level: + + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - target_label: job + replacement: kubernetes_sd + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - target_label: "job_name" + replacement: "kubernetes-logs" + - target_label: hostname + replacement: "${HOSTNAME}" + - target_label: node + source_labels: + - __meta_kubernetes_pod_node_name integrations: