---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlert
metadata:
  name: ${NAME}
  namespace: ${NAMESPACE}
spec:
  replicaCount: 1
  datasource:
    url: ${URL_DATASOURCE}
  notifier:
    url: ${URL_NOTIFIER}
  evaluationInterval: "30s"
  selectAllByDefault: true
  remoteWrite:
    url: http://vminsert-victoria-metrics-cluster.service:8480/insert/0/prometheus/

---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: vmrule-storage-iops
  namespace: ${NAMESPACE}
spec:
  groups:
    - name: storage_monitoring
      interval: 30s
      rules:
        - record: storage_class_iops_limits
          expr: |
            sum by(persistentvolume, storageclass, cluster) (
              (kube_persistentvolume_info{storageclass=~"azureblob-fuse-premium|azureblob-nfs-premium"} * 15000) or
              (kube_persistentvolume_info{storageclass=~"managed-premium|managed-csi-premium|managed-premium-burstable"} * 20000) or
              (kube_persistentvolume_info{storageclass=~"managed|managed-csi|default"} * 6000) or
              (kube_persistentvolume_info{storageclass="managed-csi-os-warm-nodes"} * 6000) or
              (kube_persistentvolume_info{storageclass=~"azurefile-premium|azurefile-csi-premium|azurefile-nonroot|azurefile-sc-fips|sc-rwm"} * 100000) or
              (kube_persistentvolume_info{storageclass=~"azurefile|azurefile-csi|azurefile-standart-nonroot"} * 10000) or
              (kube_persistentvolume_info * 500)
            )

        - record: pod_pvc_info
          expr: |
            max by(cluster, exported_namespace, pod, persistentvolumeclaim) (
              kube_pod_spec_volumes_persistentvolumeclaims_info
            )
        - record: pvc_storage_info
          expr: |
            max by(cluster, exported_namespace, persistentvolumeclaim, storageclass) (
              kube_persistentvolumeclaim_info
            )

        - record: pvc_current_iops
          expr: |
            sum by(cluster, pod, namespace, persistentvolumeclaim, storageclass) (
              (
                rate(container_fs_reads_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m]) +
                rate(container_fs_writes_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m])
              )
            )
            * on(cluster, pod) group_left(persistentvolumeclaim, exported_namespace)
            pod_pvc_info
            * on(cluster, exported_namespace, persistentvolumeclaim) group_left(storageclass)
            pvc_storage_info

        - record: pvc_iops_utilization_percent
          expr: |
            sum by(cluster, namespace, pod, persistentvolumeclaim, storageclass) (pvc_current_iops)
            * 100
            / on(cluster, storageclass) group_left()
            max by(cluster, storageclass) (storage_class_iops_limits)
---
- alert: HighIOPSUtilization
  expr: round(pvc_iops_utilization_percent) > 85
  for: 40m
  labels:
    severity: warning
  annotations:
    summary: "IOPS is `{{ $value }}%` on `{{ $labels.cluster }}`/`{{ $labels.namespace }}`/`{{ $labels.pod }}`."