From 762a9fa387be7e0cf9919075fd94705940a92a0b Mon Sep 17 00:00:00 2001 From: Jon Date: Tue, 19 Sep 2023 15:19:47 +0930 Subject: [PATCH] feat: proxy deployment for kube scheduler controller metrics !1 --- templates/ClusterRole-kube-monitor-proxy.yaml | 21 +++ ...ClusterRoleBinding-kube-monitor-proxy.yaml | 20 +++ templates/Daemonset-kube-monitor-proxy.yaml | 136 ++++++++++++++++++ templates/Service-kube-monitor-proxy.yaml | 30 ++++ .../ServiceAccount-kube-monitor-proxy.yaml | 13 ++ ...erviceMonitor-kube-controller-manager.yaml | 82 +++++++++++ templates/ServiceMonitor-kube-scheduler.yaml | 36 +++++ 7 files changed, 338 insertions(+) create mode 100644 templates/ClusterRole-kube-monitor-proxy.yaml create mode 100644 templates/ClusterRoleBinding-kube-monitor-proxy.yaml create mode 100644 templates/Daemonset-kube-monitor-proxy.yaml create mode 100644 templates/Service-kube-monitor-proxy.yaml create mode 100644 templates/ServiceAccount-kube-monitor-proxy.yaml create mode 100644 templates/ServiceMonitor-kube-controller-manager.yaml create mode 100644 templates/ServiceMonitor-kube-scheduler.yaml diff --git a/templates/ClusterRole-kube-monitor-proxy.yaml b/templates/ClusterRole-kube-monitor-proxy.yaml new file mode 100644 index 0000000..e7b364a --- /dev/null +++ b/templates/ClusterRole-kube-monitor-proxy.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-monitor-proxy + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} +rules: +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] diff --git a/templates/ClusterRoleBinding-kube-monitor-proxy.yaml b/templates/ClusterRoleBinding-kube-monitor-proxy.yaml new file mode 100644 index 0000000..0e4c1dd --- /dev/null +++ b/templates/ClusterRoleBinding-kube-monitor-proxy.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-monitor-proxy + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-monitor-proxy +subjects: +- kind: ServiceAccount + name: kube-monitor-proxy + namespace: monitoring diff --git a/templates/Daemonset-kube-monitor-proxy.yaml b/templates/Daemonset-kube-monitor-proxy.yaml new file mode 100644 index 0000000..a283196 --- /dev/null +++ b/templates/Daemonset-kube-monitor-proxy.yaml @@ -0,0 +1,136 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + + nodeExportermetricsJob: node + name_kcm: kube-controller-manager + + name: kube-monitor-proxy + namespace: "{{ .Values.nfc_monitoring.kube_monitor_proxy.namespace }}" +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + template: + metadata: + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + spec: + automountServiceAccountToken: true + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + + - args: + - --logtostderr + - --v=10 + - --secure-listen-address=[$(IP)]:11257 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - --upstream=https://127.0.0.1:10257/ + - --client-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - --upstream-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + env: + - name: NODE-IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: "{{ .Values.nfc_monitoring.kube_rbac_proxy.image.name }}:{{ .Values.nfc_monitoring.kube_rbac_proxy.image.tag }}" + name: kube-rbac-proxy-kube-ctrl-mgr + ports: + - containerPort: 10257 + #hostPort: 9100 + name: kube-ctrl-mgr + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + + + - args: + - --logtostderr + - --v=10 + - --secure-listen-address=[$(IP)]:11259 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - --upstream=https://127.0.0.1:10259/ + - --client-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - --upstream-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + env: + - name: NODE-IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: "{{ .Values.nfc_monitoring.kube_rbac_proxy.image.name }}:{{ .Values.nfc_monitoring.kube_rbac_proxy.image.tag }}" + name: kube-rbac-proxy-kube-scheduler + ports: + - containerPort: 10259 + #hostPort: 9100 + name: kube-scheduler + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + nodeSelector: + kubernetes.io/os: linux + hostPID: true + priorityClassName: system-cluster-critical + serviceAccountName: kube-monitor-proxy + tolerations: + - operator: Exists diff --git a/templates/Service-kube-monitor-proxy.yaml b/templates/Service-kube-monitor-proxy.yaml new file mode 100644 index 0000000..a23d9dd --- /dev/null +++ b/templates/Service-kube-monitor-proxy.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-monitor-proxy + namespace: monitoring + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + name_kcm: kube-controller-manager + name_ks: kube-scheduler +spec: + selector: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/part-of: {{ $.Chart.Name }} + + ports: + - name: kube-ctrl-mgr + port: 10257 + targetPort: kube-ctrl-mgr + - name: kube-scheduler + port: 10259 + targetPort: kube-scheduler + sessionAffinity: ClientIP diff --git a/templates/ServiceAccount-kube-monitor-proxy.yaml b/templates/ServiceAccount-kube-monitor-proxy.yaml new file mode 100644 index 0000000..b8e258f --- /dev/null +++ b/templates/ServiceAccount-kube-monitor-proxy.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-monitor-proxy + namespace: monitoring + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} diff --git a/templates/ServiceMonitor-kube-controller-manager.yaml b/templates/ServiceMonitor-kube-controller-manager.yaml new file mode 100644 index 0000000..21d30f2 --- /dev/null +++ b/templates/ServiceMonitor-kube-controller-manager.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-controller-manager + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + name_kcm: kube-controller-manager + name: kube-controller-manager + namespace: monitoring +spec: + endpoints: + + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 10s + honorLabels: true + path: /metrics + port: kube-ctrl-mgr + scheme: https + # labels: + # job: kube-controller-manager + relabelings: + # - action: replace + # regex: (.*) + # replacement: $1 + # sourceLabels: + # - __meta_kubernetes_pod_node_name + # targetLabel: instance + - action: replace + regex: (.*) + replacement: $1 + sourceLabels: + - __meta_kubernetes_service_label_name_kcm + targetLabel: job + metricRelabelings: + - action: drop + regex: kubelet_(pod_worker_latency_microseconds|pod_start_latency_microseconds|cgroup_manager_latency_microseconds|pod_worker_start_latency_microseconds|pleg_relist_latency_microseconds|pleg_relist_interval_microseconds|runtime_operations|runtime_operations_latency_microseconds|runtime_operations_errors|eviction_stats_age_microseconds|device_plugin_registration_count|device_plugin_alloc_latency_microseconds|network_plugin_operations_latency_microseconds) + sourceLabels: + - __name__ + - action: drop + regex: scheduler_(e2e_scheduling_latency_microseconds|scheduling_algorithm_predicate_evaluation|scheduling_algorithm_priority_evaluation|scheduling_algorithm_preemption_evaluation|scheduling_algorithm_latency_microseconds|binding_latency_microseconds|scheduling_latency_seconds) + sourceLabels: + - __name__ + - action: drop + regex: apiserver_(request_count|request_latencies|request_latencies_summary|dropped_requests|storage_data_key_generation_latencies_microseconds|storage_transformation_failures_total|storage_transformation_latencies_microseconds|proxy_tunnel_sync_latency_secs|longrunning_gauge|registered_watchers) + sourceLabels: + - __name__ + - action: drop + regex: kubelet_docker_(operations|operations_latency_microseconds|operations_errors|operations_timeout) + sourceLabels: + - __name__ + - action: drop + regex: reflector_(items_per_list|items_per_watch|list_duration_seconds|lists_total|short_watches_total|watch_duration_seconds|watches_total) + sourceLabels: + - __name__ + - action: drop + regex: etcd_(helper_cache_hit_count|helper_cache_miss_count|helper_cache_entry_count|object_counts|request_cache_get_latencies_summary|request_cache_add_latencies_summary|request_latencies_summary) + sourceLabels: + - __name__ + - action: drop + regex: transformation_(transformation_latencies_microseconds|failures_total) + sourceLabels: + - __name__ + - action: drop + regex: (admission_quota_controller_adds|admission_quota_controller_depth|admission_quota_controller_longest_running_processor_microseconds|admission_quota_controller_queue_latency|admission_quota_controller_unfinished_work_seconds|admission_quota_controller_work_duration|APIServiceOpenAPIAggregationControllerQueue1_adds|APIServiceOpenAPIAggregationControllerQueue1_depth|APIServiceOpenAPIAggregationControllerQueue1_longest_running_processor_microseconds|APIServiceOpenAPIAggregationControllerQueue1_queue_latency|APIServiceOpenAPIAggregationControllerQueue1_retries|APIServiceOpenAPIAggregationControllerQueue1_unfinished_work_seconds|APIServiceOpenAPIAggregationControllerQueue1_work_duration|APIServiceRegistrationController_adds|APIServiceRegistrationController_depth|APIServiceRegistrationController_longest_running_processor_microseconds|APIServiceRegistrationController_queue_latency|APIServiceRegistrationController_retries|APIServiceRegistrationController_unfinished_work_seconds|APIServiceRegistrationController_work_duration|autoregister_adds|autoregister_depth|autoregister_longest_running_processor_microseconds|autoregister_queue_latency|autoregister_retries|autoregister_unfinished_work_seconds|autoregister_work_duration|AvailableConditionController_adds|AvailableConditionController_depth|AvailableConditionController_longest_running_processor_microseconds|AvailableConditionController_queue_latency|AvailableConditionController_retries|AvailableConditionController_unfinished_work_seconds|AvailableConditionController_work_duration|crd_autoregistration_controller_adds|crd_autoregistration_controller_depth|crd_autoregistration_controller_longest_running_processor_microseconds|crd_autoregistration_controller_queue_latency|crd_autoregistration_controller_retries|crd_autoregistration_controller_unfinished_work_seconds|crd_autoregistration_controller_work_duration|crdEstablishing_adds|crdEstablishing_depth|crdEstablishing_longest_running_processor_microseconds|crdEstablishing_queue_latency|crdEstablishing_retries|crdEstablishing_unfinished_work_seconds|crdEstablishing_work_duration|crd_finalizer_adds|crd_finalizer_depth|crd_finalizer_longest_running_processor_microseconds|crd_finalizer_queue_latency|crd_finalizer_retries|crd_finalizer_unfinished_work_seconds|crd_finalizer_work_duration|crd_naming_condition_controller_adds|crd_naming_condition_controller_depth|crd_naming_condition_controller_longest_running_processor_microseconds|crd_naming_condition_controller_queue_latency|crd_naming_condition_controller_retries|crd_naming_condition_controller_unfinished_work_seconds|crd_naming_condition_controller_work_duration|crd_openapi_controller_adds|crd_openapi_controller_depth|crd_openapi_controller_longest_running_processor_microseconds|crd_openapi_controller_queue_latency|crd_openapi_controller_retries|crd_openapi_controller_unfinished_work_seconds|crd_openapi_controller_work_duration|DiscoveryController_adds|DiscoveryController_depth|DiscoveryController_longest_running_processor_microseconds|DiscoveryController_queue_latency|DiscoveryController_retries|DiscoveryController_unfinished_work_seconds|DiscoveryController_work_duration|kubeproxy_sync_proxy_rules_latency_microseconds|non_structural_schema_condition_controller_adds|non_structural_schema_condition_controller_depth|non_structural_schema_condition_controller_longest_running_processor_microseconds|non_structural_schema_condition_controller_queue_latency|non_structural_schema_condition_controller_retries|non_structural_schema_condition_controller_unfinished_work_seconds|non_structural_schema_condition_controller_work_duration|rest_client_request_latency_seconds|storage_operation_errors_total|storage_operation_status_count) + sourceLabels: + - __name__ + - action: drop + regex: etcd_(debugging|disk|request|server).* + sourceLabels: + - __name__ + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/part-of: {{ $.Chart.Name }} diff --git a/templates/ServiceMonitor-kube-scheduler.yaml b/templates/ServiceMonitor-kube-scheduler.yaml new file mode 100644 index 0000000..71149ce --- /dev/null +++ b/templates/ServiceMonitor-kube-scheduler.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-scheduler + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/part-of: {{ $.Chart.Name }} + app.kubernetes.io/version: {{ $.Chart.Version }} + name: kube-scheduler + namespace: monitoring +spec: + endpoints: + + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 10s + honorLabels: true + path: /metrics + port: kube-scheduler + scheme: https + relabelings: + - action: replace + regex: (.*) + replacement: $1 + sourceLabels: + - __meta_kubernetes_service_label_name_ks + targetLabel: job + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: kube-monitor-proxy + app.kubernetes.io/part-of: {{ $.Chart.Name }}