Adds Monitoring for the Prometheus & Grafana Operators

kustomize/monitoring/operator/grafana/dashboards.yaml

@@ -0,0 +1,95 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: pgbackrest
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: pgbackrest.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: pgbouncer-direct
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: pgbouncer_direct.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: pod-details
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: pod_details.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: postgresql-details
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: postgresql_details.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: postgresql-overview
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: postgresql_overview.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: postgresql-service-health
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: postgresql_service_health.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: prometheus-alerts
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: prometheus_alerts.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: query-statistics
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  configMapRef:
+    name: grafana-dashboards
+    key: query_statistics.json

kustomize/monitoring/operator/grafana/datasources.yaml

@@ -0,0 +1,27 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: grafana-datasources
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: crunchy-grafana
+  valuesFrom:
+  - targetPath: "secureJsonData.httpHeaderValue1"
+    valueFrom:
+      secretKeyRef:
+        name: grafana-pgo
+        key: token
+  datasource:
+    access: proxy
+    editable: true
+    isDefault: true
+    jsonData:
+      httpHeaderName1: 'Authorization'
+      timeInterval: 5s
+      tlsSkipVerify: true
+    name: PROMETHEUS
+    secureJsonData:
+      httpHeaderValue1: 'Bearer ${token}'
+    type: prometheus
+    url: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091/

kustomize/monitoring/operator/grafana/grafana.yaml

@@ -0,0 +1,24 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: Grafana
+metadata:
+  name: crunchy-grafana
+  labels:
+    dashboards: crunchy-grafana
+spec:
+  deployment:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: grafana
+            env:
+            - name: GF_SECURITY_ADMIN_USER
+              valueFrom:
+                secretKeyRef:
+                  name: grafana-admin
+                  key: username
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: grafana-admin
+                  key: password

kustomize/monitoring/operator/grafana/kustomization.yaml

@@ -0,0 +1,17 @@
+generatorOptions:
+  disableNameSuffixHash: true
+
+secretGenerator:
+- name: grafana-admin
+  literals:
+  - password=admin
+  - username=admin
+  type: Opaque
+
+resources:
+- rbac.yaml
+- grafana.yaml
+- datasources.yaml
+- ../../grafana/dashboards
+- dashboards.yaml
+

kustomize/monitoring/operator/grafana/rbac.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana-pgo
+---
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/service-account-token
+metadata:
+  name: grafana-pgo
+  annotations:
+    kubernetes.io/service-account.name: grafana-pgo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: grafana-pgo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-monitoring-view
+subjects:
+- kind: ServiceAccount
+  name: grafana-pgo
+  namespace: postgres-operator

kustomize/monitoring/operator/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: postgres-operator
+
+resources:
+- grafana
+- prometheus

kustomize/monitoring/operator/prometheus/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- rbac.yaml
+- podmonitor.yaml

kustomize/monitoring/operator/prometheus/podmonitor.yaml

@@ -0,0 +1,86 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: crunchy-otel-collector
+spec:
+  selector:
+    matchLabels:
+      postgres-operator.crunchydata.com/crunchy-otel-collector: "true"
+
+  # Uncomment to monitor a single namespace only      
+  # namespaceSelector:
+  #   matchNames:
+  #     - postgres-operator
+
+  podMetricsEndpoints:
+  - port: otel-metrics
+    interval: 15s
+    scrapeTimeout: 15s
+    relabelings:
+      # Keep exporter port and drop all others
+      - sourceLabels: [__meta_kubernetes_pod_container_port_number]
+        action: keep
+        regex: "9187"
+      # Set label for namespace
+      - sourceLabels: [__meta_kubernetes_namespace]
+        targetLabel: kubernetes_namespace
+      # Set label for pod name
+      - sourceLabels: [__meta_kubernetes_pod_name]
+        targetLabel: pod
+      # Convert namespace and cluster name to pg_cluster=namespace:cluster
+      - sourceLabels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_cluster]
+        targetLabel: pg_cluster
+        separator: ":"
+        replacement: '$1$2'
+      # Convert kubernetes pod ip to ip
+      - sourceLabels: [__meta_kubernetes_pod_ip]
+        targetLabel: ip
+      # Convert postgres-operator.crunchydata.com/instance to deployment
+      - sourceLabels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_instance]
+        targetLabel: deployment
+      # Convert postgres-operator.crunchydata.com/role to role
+      - sourceLabels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_role]
+        targetLabel: role
+      # The following relabels should make it easier to use pgMonitor dashboards.
+      # Note: The following was added for the pgBouncer dashboard and what labels it requires.
+      # For pgBouncer, `exp_type` should be equal to role.
+      - sourceLabels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_role]
+        targetLabel: exp_type
+      # `cluster_name` is equivalent to `pg_cluster`
+      - sourceLabels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_cluster]
+        targetLabel: cluster_name
+        separator: ":"
+        replacement: '$1$2'
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: pgo-metrics
+spec:
+  selector:
+    matchExpressions:
+    - key: postgres-operator.crunchydata.com/control-plane
+      operator: Exists
+
+  # Uncomment to monitor a single namespace only      
+  # namespaceSelector:
+  #   matchNames:
+  #     - postgres-operator
+
+  podMetricsEndpoints:
+  - port: metrics
+    interval: 15s
+    scrapeTimeout: 15s
+    # If you are running CPK v5.7 or earlier, you will need to change the scheme to 'http'
+    # and add a metrics port to the postgres-operator deployment that exposes port 8080.
+    scheme: https
+    authorization:
+      type: Bearer
+      credentials:
+        name: prometheus-pgo
+        key: token
+    tlsConfig:
+      # By default, the operator's metrics server automatically creates self-signed certs
+      # which cannot be verified, so `insecure_skip_verify` is set to `true`. See the
+      # documentation for providing your own signed certificates.
+      insecureSkipVerify: true

kustomize/monitoring/operator/prometheus/rbac.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-pgo
+---
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/service-account-token
+metadata:
+  name: prometheus-pgo
+  annotations:
+    kubernetes.io/service-account.name: prometheus-pgo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-pgo
+rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-pgo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-pgo
+subjects:
+- kind: ServiceAccount
+  name: prometheus-pgo
+  namespace: postgres-operator