From e0bc34c12ff88b8e75806dc238b00e1ef464129b Mon Sep 17 00:00:00 2001 From: Jon Date: Tue, 19 Sep 2023 15:12:04 +0930 Subject: [PATCH] feat: grafana agent deployment !1 --- templates/ClusterRole-GrafanaAgent.yaml | 42 +++ .../ClusterRoleBinding-Grafana-Agent.yaml | 20 ++ templates/ConfigMap-GrafanaAgent.yaml | 281 ++++++++++++++++++ templates/Daemonset-GrafanaAgent.yaml | 136 +++++++++ templates/PrometheusRule-grafana-agent.yaml | 23 ++ templates/Service-GrafanaAgent.yaml | 28 ++ templates/ServiceAccount-GrafanaAgent.yaml | 14 + 7 files changed, 544 insertions(+) create mode 100644 templates/ClusterRole-GrafanaAgent.yaml create mode 100644 templates/ClusterRoleBinding-Grafana-Agent.yaml create mode 100644 templates/ConfigMap-GrafanaAgent.yaml create mode 100644 templates/Daemonset-GrafanaAgent.yaml create mode 100644 templates/PrometheusRule-grafana-agent.yaml create mode 100644 templates/Service-GrafanaAgent.yaml create mode 100644 templates/ServiceAccount-GrafanaAgent.yaml diff --git a/templates/ClusterRole-GrafanaAgent.yaml b/templates/ClusterRole-GrafanaAgent.yaml new file mode 100644 index 0000000..a5886ab --- /dev/null +++ b/templates/ClusterRole-GrafanaAgent.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: grafana-agent + labels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + - events + verbs: + - get + - list + - watch +- nonResourceURLs: + - /metrics + verbs: + - get +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/templates/ClusterRoleBinding-Grafana-Agent.yaml b/templates/ClusterRoleBinding-Grafana-Agent.yaml new file mode 100644 index 0000000..403db42 --- /dev/null +++ b/templates/ClusterRoleBinding-Grafana-Agent.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + name: grafana-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grafana-agent +subjects: +- kind: ServiceAccount + name: grafana-agent + namespace: monitoring diff --git a/templates/ConfigMap-GrafanaAgent.yaml b/templates/ConfigMap-GrafanaAgent.yaml new file mode 100644 index 0000000..d866aee --- /dev/null +++ b/templates/ConfigMap-GrafanaAgent.yaml @@ -0,0 +1,281 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + name: grafana-agent + namespace: monitoring +data: + agent.yaml: | + metrics: + wal_directory: /tmp/wal + + logs: + # Choose a directory to save the last read position of log files at. + # This directory will be created if it doesn't already exist. + positions_directory: "/tmp" + + configs: + + - name: journal + clients: + - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:3100/loki/api/v1/push + scrape_configs: + + - job_name: systemd-journal + journal: + labels: + job: node-journal + path: /host/root/run/log/journal + json: true + relabel_configs: + + - source_labels: + - __journal__systemd_unit + target_label: systemd_unit + - source_labels: + - __journal__hostname + target_label: node + - source_labels: + - __journal_syslog_identifier + target_label: syslog_identifier + - target_label: "job_name" + replacement: "journal" + + pipeline_stages: + - json: + expressions: + pid: _PID + userId: _UID + application: _COMM + priority: PRIORITY + + - labels: + application: + #level: + pid: + userId: + priority: + - template: + source: level + template: '{{"{{"}} ToLower .Value {{"}}"}}' + - match: + selector: '{priority="7"}' + stages: + - template: + source: level + template: 'debug' + - match: + selector: '{priority="6"}' + stages: + - template: + source: level + template: 'info' + - match: + selector: '{priority="5"}' + stages: + - template: + source: level + template: 'notice' + - match: + selector: '{priority="4"}' + stages: + - template: + source: level + template: 'warning' + - match: + selector: '{priority="3"}' + stages: + - template: + source: level + template: 'error' + - match: + selector: '{priority="2"}' + stages: + - template: + source: level + template: 'crit' + - match: + selector: '{priority="1"}' + stages: + - template: + source: level + template: 'alert' + - match: + selector: '{priority="0"}' + stages: + - template: + source: level + template: 'emerg' + - labels: + level: + + # - job_name: varlogs + # static_configs: + # - targets: [localhost] + # labels: + # # cluster: dev + # job: 'container logs' + # __path__: /var/log/pods/*/*/*.log + # pipeline_stages: + # - json: + # expressions: + # namespace: namespace + # level: level + # - regex: + # source: filename + # # expression: '/var/log/pods/(.+)/(?P.+)/(*.log)' + # # expression: '/var/log/pods/.+/(?P\\S+?)' + # #expression: '/var/log/pods/.+/(?P\\S+?)/*.log' + # #expression: '/var/log/pods/.+/(?P\S+?)/*.log' + # expression: '/var/log/pods/.+/(?P\S+?)/.+log' + # - regex: + # source: filename + # expression: '/var\/log\/pods\/.*_(?P\S+?)_.*\/.+\/.+log' + # - regex: + # source: filename + # expression: '/var/log/pods/(?P\S+?)_.*/.+/.+log' + # - regex: + # #source: message + # expression: '\s?level=(?P\S+)\s+' + # - regex: + # #source: message + # expression: '\s?(?Pdebug|info|warn|error)\s' + # - template: + # source: level + # template: '{{"{{"}} ToLower .Value {{"}}"}}' + # - labels: + # container: + # level: + # namespace: + # pod: + #relabel_configs: + + # - source_labels: [__filename__] + # separator: '/' + # regex: '/var/log/pods/*/(.*)/*.log' + # replacement: '${1}' + # target_label: pod + # - source_labels: [__filename__] + # replacement: '/var/log/pods/*/${1}/*.log' + # target_label: pod + # - replacement: /var/log/pods/*/${1}/*.log + # separator: / + # source_labels: + # - __meta_kubernetes_pod_uid + # - __meta_kubernetes_pod_container_name + # target_label: pod + + + + # - targets: [localhost] + # labels: + # job: varlogs + # __path__: /var/logs/* + # - targets: [localhost] + # labels: + # job: varlogs + # __path__: /host/root/var/log/* + # - targets: [localhost] + # labels: + # job: varlogs + # __path__: /host/root/var/logs/* + # - targets: [localhost] + # labels: + # job: varlogs + # __path__: /host/root/var/log/containers/* + # - targets: [localhost] + # labels: + # job: varlogs + # __path__: /host/root/var/log/*/* + + + + + + + - name: kubernetes + clients: + - url: http://{{ .Values.nfc_monitoring.loki.service_name }}.{{ .Values.nfc_monitoring.loki.namespace }}.svc.{{ .Values.nfc_monitoring.kubernetes.cluster_dns_name }}:3100/loki/api/v1/push + # basic_auth: + # username: YOUR_LOKI_USERNAME + # password: YOUR_LOKI_PASSWORD + external_labels: + #cluster: dev-cluster + positions: + filename: /tmp/kub-positions.yaml + target_config: + sync_period: 10s + scrape_configs: + - job_name: pod-logs + + kubernetes_sd_configs: + - role: pod + + pipeline_stages: + - cri: {} + + - regex: + #source: msg + expression: '(\s|\t|level=)?(?Ptrace|debug|info|warn|error|TRACE|DEBUG|INFO|WARN|ERROR)(\s|\t)' + + - labels: + level: + + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - target_label: job + # source_labels: + # - __meta_kubernetes_namespace + replacement: loki/ingester_grafana-agent + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - target_label: "job_name" + replacement: "kubernetes_sd" + + + + integrations: + + node_exporter: + enabled: true + rootfs_path: /host/root + sysfs_path: /host/sys + procfs_path: /host/proc + udev_data_path: /host/root/run/udev/data + + # collector.filesystem.ignored-mount-points: ^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+|/run/containerd/io.containerd.+)($|/) + filesystem_mount_points_exclude: "^/(dev|proc|sys|var/lib/docker/.+|/run/containerd/io.containerd.+)($|/)" + filesystem_fs_types_exclude: "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|shm|squashfs|sysfs|tracefs)$" + + scrape_integration: true + + include_exporter_metrics: true + enable_collectors: + - uname diff --git a/templates/Daemonset-GrafanaAgent.yaml b/templates/Daemonset-GrafanaAgent.yaml new file mode 100644 index 0000000..b04a6c3 --- /dev/null +++ b/templates/Daemonset-GrafanaAgent.yaml @@ -0,0 +1,136 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + metricsJob: node-exporter + cadvisormetricsJob: cadvisor + nodeExportermetricsJob: node + name: grafana-agent + namespace: "{{ .Values.nfc_monitoring.grafana_agent.namespace }}" +spec: + selector: + matchLabels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + metricsJob: node-exporter + cadvisormetricsJob: cadvisor + nodeExportermetricsJob: node + template: + metadata: + labels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + metricsJob: node-exporter + cadvisormetricsJob: cadvisor + nodeExportermetricsJob: node + spec: + automountServiceAccountToken: true + containers: + - args: + - --server.http.address=0.0.0.0:12345 + - --config.file=/etc/agent/agent.yaml + name: grafana-agent + image: "{{ .Values.nfc_monitoring.grafana_agent.image.name }}:{{ .Values.nfc_monitoring.grafana_agent.image.tag }}" + #imagePullPolicy: Never + ports: + - containerPort: 12345 + name: grafana-metrics + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 180Mi + requests: + cpu: 40m + memory: 180Mi + securityContext: + capabilities: + add: + - SYS_TIME + # drop: + # - ALL + readOnlyRootFilesystem: false + privileged: true + volumeMounts: + - mountPath: /host/sys + mountPropagation: HostToContainer + name: sys + readOnly: true + - mountPath: /host/proc + mountPropagation: HostToContainer + name: proc + readOnly: true + - mountPath: /host/root + mountPropagation: HostToContainer + name: rootfs + readOnly: true + - mountPath: /var/log + mountPropagation: HostToContainer + name: logs + readOnly: true + - name: config + mountPath: "/etc/agent" + readOnly: false + - name: temp + mountPath: "/tmp" + readOnly: false + - name: agent-data + mountPath: "/etc/agent/data" + readOnly: false + volumes: + - hostPath: + path: /sys + name: sys + - hostPath: + path: /proc + name: proc + - hostPath: + path: / + name: rootfs + - hostPath: + path: /var/log + name: logs + - name: config + configMap: + name: grafana-agent + items: + - key: "agent.yaml" + path: "agent.yaml" + - name: temp + emptyDir: {} + - name: agent-data + emptyDir: {} + + - name: var-run + hostPath: + path: /var/run + - name: containerd + hostPath: + path: /var/lib/contairnerd + - name: disk + hostPath: + path: /dev/disk + + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + serviceAccountName: grafana-agent + tolerations: + - operator: Exists diff --git a/templates/PrometheusRule-grafana-agent.yaml b/templates/PrometheusRule-grafana-agent.yaml new file mode 100644 index 0000000..6467f12 --- /dev/null +++ b/templates/PrometheusRule-grafana-agent.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + app.kubernetes.io/managed-by: {{ $.Release.Service }} + prometheus: k8s + role: grafana-agent-promtail + name: grafana-agent + namespace: monitoring +spec: + groups: + - name: grafana_agent + rules: + # - annotations: + # description: "As Grafana Agent is being used, it's version is set as promtails" + - expr: | + agent_build_info + record: promtail_build_info diff --git a/templates/Service-GrafanaAgent.yaml b/templates/Service-GrafanaAgent.yaml new file mode 100644 index 0000000..fc070ab --- /dev/null +++ b/templates/Service-GrafanaAgent.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana-agent + namespace: monitoring + labels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} +spec: + selector: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/part-of: {{ $.Chart.Name }} + ports: + - name: grafana-metrics + port: 12345 + targetPort: grafana-metrics + - name: kube-ctrl-mgr + port: 11257 + targetPort: kube-ctrl-mgr + #type: LoadBalancer + sessionAffinity: ClientIP diff --git a/templates/ServiceAccount-GrafanaAgent.yaml b/templates/ServiceAccount-GrafanaAgent.yaml new file mode 100644 index 0000000..32363be --- /dev/null +++ b/templates/ServiceAccount-GrafanaAgent.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +automountServiceAccountToken: false +kind: ServiceAccount +metadata: + name: grafana-agent + namespace: monitoring + labels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/component: exporter + app.kubernetes.io/name: grafana-agent + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }}