概述

GitHub地址：https://github.com/prometheus-operator/prometheus-operator
Helm倉庫：https://prometheus-community.github.io/helm-charts

安裝prometheus-operator前置條件：

確保你有一個k8s集群

安裝k8s集群可參考：使用sealos快速搭建kubernetes集群！！！

確保你安裝了Helm

安裝Helm可參考：安裝Helm

實操

創建SC

我這里以NFS作為存儲

配置NFS參考這篇文章：http://www.rzrgm.cn/huangSir-devops/p/18802386
創建SC參考：http://www.rzrgm.cn/huangSir-devops/p/18871414#_label4_2

創建grafana的存儲

# 配置NFS
[root@master01 ~/volumes]# mkdir -p /data/nfs/grafana
[root@master01 ~/volumes]# echo '/data/nfs/grafana 10.0.0.0/24(rw,sync,no_root_squash,no_subtree_check)' >> /etc/exports
[root@master ~/volumes]# exportfs -ra

# 定義資源清單
[root@master01 ~/volumes]# cat grafana-sc.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: grafana-sc  # StorageClass名稱，PVC通過該名稱引用此存儲類
provisioner: nfs.csi.k8s.io  # 指定使用NFS CSI驅動作為存儲供給器
parameters:  # 傳遞給NFS CSI驅動的參數
  server: 10.0.0.30  # NFS服務器的IP地址
  share: /data/nfs/grafana  # NFS服務器上的共享目錄路徑
  # csi.storage.k8s.io/provisioner-secret is only needed for providing mountOptions in DeleteVolume
  # csi.storage.k8s.io/provisioner-secret-name: "mount-options"
  # csi.storage.k8s.io/provisioner-secret-namespace: "default"
reclaimPolicy: Retain  # 回收策略：當PVC被刪除時，PV保留不刪除
volumeBindingMode: Immediate  # 卷綁定模式：立即綁定，不需要等待Pod調度
allowVolumeExpansion: true  # 允許卷擴容：支持通過修改PVC請求更大容量

[root@master ~]# kubectl apply -f grafana-sc.yaml

[root@master ~]# kubectl get sc
NAME         PROVISIONER      RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
grafana-sc   nfs.csi.k8s.io   Retain          Immediate           true                   3m44s

創建Prometheus的存儲

# 配置NFS
[root@master01 ~/volumes]# mkdir -p /data/nfs/prometheus
[root@master01 ~/volumes]# echo '/data/nfs/prometheus 10.0.0.0/24(rw,sync,no_root_squash,no_subtree_check)' >> /etc/exports
[root@master ~/volumes]# exportfs -ra

[root@master01 ~/volumes]# cat prometheus-sc.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: prometheus-sc  # StorageClass名稱，PVC通過該名稱引用此存儲類
provisioner: nfs.csi.k8s.io  # 指定使用NFS CSI驅動作為存儲供給器
parameters:  # 傳遞給NFS CSI驅動的參數
  server: 10.0.0.30  # NFS服務器的IP地址
  share: /data/nfs/prometheus  # NFS服務器上的共享目錄路徑
  # csi.storage.k8s.io/provisioner-secret is only needed for providing mountOptions in DeleteVolume
  # csi.storage.k8s.io/provisioner-secret-name: "mount-options"
  # csi.storage.k8s.io/provisioner-secret-namespace: "default"
reclaimPolicy: Retain  # 回收策略：當PVC被刪除時，PV保留不刪除
volumeBindingMode: Immediate  # 卷綁定模式：立即綁定，不需要等待Pod調度
allowVolumeExpansion: true  # 允許卷擴容：支持通過修改PVC請求更大容量

[root@master ~/volumes]# kubectl apply -f prometheus-sc.yaml
storageclass.storage.k8s.io/prometheus-sc created
[root@master ~/volumes]# kubectl get sc
NAME            PROVISIONER      RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
grafana-sc      nfs.csi.k8s.io   Retain          Immediate           true                   3h40m
nfs-csi         nfs.csi.k8s.io   Delete          Immediate           true                   175m
prometheus-sc   nfs.csi.k8s.io   Retain          Immediate           true                   4s

配置operator

添加倉庫

# 添加社區版倉庫
[root@master ~]# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
[root@master ~]# helm repo update
[root@master ~]# helm pull helm pull prometheus/kube-prometheus-stack --version 73.2.0


# 如果上面因為網絡報錯，可使用下面的
[root@master ~]# wget https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-73.2.0/kube-prometheus-stack-73.2.0.tgz

解壓

[root@master ~]# tar -xvf kube-prometheus-stack-73.2.0.tgz
[root@master ~]# ll kube-prometheus-stack
total 228
drwxr-xr-x  4 root root   4096 Jun  7 15:27 ./
drwx------ 31 root root   4096 Jun  7 15:27 ../
-rw-r--r--  1 root root    458 Jun  6 01:22 .helmignore
-rw-r--r--  1 root root    615 Jun  6 01:22 Chart.lock
-rw-r--r--  1 root root   2571 Jun  6 01:22 Chart.yaml
-rw-r--r--  1 root root  19869 Jun  6 01:22 README.md
drwxr-xr-x  7 root root   4096 Jun  7 15:27 charts/
drwxr-xr-x  8 root root   4096 Jun  7 15:27 templates/
-rw-r--r--  1 root root 182959 Jun  6 01:22 values.yaml

修改鏡像源，防止鏡像拉取失敗

[root@master ~/kube-prometheus-stack]# sed -i s#registry.k8s.io#k8s.m.daocloud.io#g values.yaml
[root@master ~/kube-prometheus-stack]# sed -i s#quay.io#quay.m.daocloud.io#g values.yaml
[root@master ~/kube-prometheus-stack]# sed -i s#docker.io#docker.m.daocloud.io#g values.yaml
[root@master ~/kube-prometheus-stack]# sed -i s#registry.k8s.io#k8s.m.daocloud.io#g charts/kube-state-metrics/values.yaml

配置Grafana

[root@master ~]# vim ~/kube-prometheus-stack/charts/grafana/values.yaml
# 修改service，改成NodePort訪問
 231 service:
 232   enabled: true
 233   type: NodePort
 234   nodePort: 30030

# 配置存儲
 417 persistence:
 418   type: pvc
 419   enabled: true
       # sc的名稱，和上面保持一致
 420   storageClassName: grafana-sc
 421   ## (Optional) Use this to bind the claim to an existing PersistentVolume (PV) by name.
 422   volumeName: ""

# 修改用戶名密碼 （可選）
[root@master ~]# vim ~/kube-prometheus-stack/values.yaml
1287   adminUser: admin
1288   adminPassword: prom-operator

配置Prometheus

[root@master ~]# vim ~/kube-prometheus-stack/values.yaml
3371 prometheus:
3372   enabled: true
# 修改service類型NodePort
3564    type: NodePort
# 端口可選配置
3551     nodePort: 30090
# 存儲配置
4299     storageSpec:
4300     ## Using PersistentVolumeClaim
4301     ##
4302       volumeClaimTemplate:
4303         spec:
4304           storageClassName: prometheus-sc
4305           accessModes: ["ReadWriteOnce"]
4306           resources:
4307             requests:
4308               storage: 10Gi

創建Realsese

[root@master ~]# helm install prometheus kube-prometheus-stack --debug
# 出現以下內容即代表成功
NOTES:
kube-prometheus-stack has been installed. Check its status by running:
  kubectl --namespace default get pods -l "release=prometheus"

Get Grafana 'admin' user password by running:

  kubectl --namespace default get secrets prometheus-grafana -o jsonpath="{.data.admin-password}" | base64 -d ; echo

Access Grafana local instance:

  export POD_NAME=$(kubectl --namespace default get pod -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=prometheus" -oname)
  kubectl --namespace default port-forward $POD_NAME 3000

Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator.

查看創建的資源

# 查看Pod
[root@master ~]# kubectl get po
NAME                                                     READY   STATUS    RESTARTS   AGE
alertmanager-prometheus-kube-prometheus-alertmanager-0   2/2     Running   0          70s
prometheus-grafana-55cbbf54b7-lmhnd                      3/3     Running   0          70s
prometheus-kube-prometheus-operator-847fd659bc-scp4w     1/1     Running   0          70s
prometheus-kube-state-metrics-5fb66759db-nb242           1/1     Running   0          70s
prometheus-prometheus-kube-prometheus-prometheus-0       2/2     Running   0          70s
prometheus-prometheus-node-exporter-89xt7                1/1     Running   0          70s
prometheus-prometheus-node-exporter-cn8s4                1/1     Running   0          70s
prometheus-prometheus-node-exporter-llqgx                1/1     Running   0          70s

# 查看svc
[root@master ~]# kubectl get svc
NAME                                      TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)                         AGE
alertmanager-operated                     ClusterIP   None          <none>        9093/TCP,9094/TCP,9094/UDP      5m
kubernetes                                ClusterIP   10.96.0.1     <none>        443/TCP                         5m
prometheus-grafana                        NodePort    10.96.1.173   <none>        80:30030/TCP                    5m
prometheus-kube-prometheus-alertmanager   ClusterIP   10.96.3.151   <none>        9093/TCP,8080/TCP               5m
prometheus-kube-prometheus-operator       ClusterIP   10.96.2.62    <none>        443/TCP                         5m
prometheus-kube-prometheus-prometheus     NodePort    10.96.0.114   <none>        9090:30090/TCP,8080:31659/TCP   5m
prometheus-kube-state-metrics             ClusterIP   10.96.1.88    <none>        8080/TCP                        5m
prometheus-operated                       ClusterIP   None          <none>        9090/TCP                        5m
prometheus-prometheus-node-exporter       ClusterIP   10.96.0.191   <none>        9100/TCP                        5m

訪問測試

訪問Grafana

http://10.0.0.30:30030/

• 用戶名：admin
• 密碼：prom-operator

用戶名和密碼是上面配置的

訪問Prometheus

http://10.0.0.30:30090/targets

kube-prometheus-stack監控應用程序

kube-prometheus-stack通過PodMonitor和ServiceMonitor對象來監視示例應用程序的目標

參考這篇文章：https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/developer/getting-started.md

架構圖

通過ServiceMonitor監控應用程序

示例：

省略，這塊有時間補齊，可以先看官方文檔

通過PodMonitor監控應用程序

示例：

省略，這塊有時間補齊，可以先看官方文檔