Make. Build. Break. Reflect.

apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
...
spec:
  groups:
...
    - name: "alertmanager"
      rules:
        - alert: Watchdog
          expr: vector(1)
          for: 5m
          labels:
            severity: info
          annotations:
            summary: "This is an Watchdog alert that is always firing."
...

route:
...
  routes:
    ##############################################################
    #                          Watchdog                          #
    ##############################################################
    # DO NOT REMOVE THIS ROUTE!!!
    # DO NOT ADD ROUTES ABOVE!!!
    - receiver: 'webhook-receiver'
      group_wait: 10s
      group_interval: 30s
      repeat_interval: 30s
      matchers:
        - alertname = "Watchdog"
      continue: true
...
receivers:
...
  - name: 'webhook-receiver'
    webhook_configs:
      - url: 'https://cross-alerting.domain.io/api/alert'
        send_resolved: true
....

---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlert
metadata:
  name: ${NAME}
  namespace: ${NAMESPACE}
spec:
  replicaCount: 1
  datasource:
    url: ${URL_DATASOURCE}
  notifier:
    url: ${URL_NOTIFIER}
  evaluationInterval: "30s"
  selectAllByDefault: true
  remoteWrite:
    url: http://vminsert-victoria-metrics-cluster.service:8480/insert/0/prometheus/

---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: vmrule-storage-iops
  namespace: ${NAMESPACE}
spec:
  groups:
    - name: storage_monitoring
      interval: 30s
      rules:
        - record: storage_class_iops_limits
          expr: |
            sum by(persistentvolume, storageclass, cluster) (
              (kube_persistentvolume_info{storageclass=~"azureblob-fuse-premium|azureblob-nfs-premium"} * 15000) or
              (kube_persistentvolume_info{storageclass=~"managed-premium|managed-csi-premium|managed-premium-burstable"} * 20000) or
              (kube_persistentvolume_info{storageclass=~"managed|managed-csi|default"} * 6000) or
              (kube_persistentvolume_info{storageclass="managed-csi-os-warm-nodes"} * 6000) or
              (kube_persistentvolume_info{storageclass=~"azurefile-premium|azurefile-csi-premium|azurefile-nonroot|azurefile-sc-fips|sc-rwm"} * 100000) or
              (kube_persistentvolume_info{storageclass=~"azurefile|azurefile-csi|azurefile-standart-nonroot"} * 10000) or
              (kube_persistentvolume_info * 500)
            )

        - record: pod_pvc_info
          expr: |
            max by(cluster, exported_namespace, pod, persistentvolumeclaim) (
              kube_pod_spec_volumes_persistentvolumeclaims_info
            )
        - record: pvc_storage_info
          expr: |
            max by(cluster, exported_namespace, persistentvolumeclaim, storageclass) (
              kube_persistentvolumeclaim_info
            )

        - record: pvc_current_iops
          expr: |
            sum by(cluster, pod, namespace, persistentvolumeclaim, storageclass) (
              (
                rate(container_fs_reads_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m]) +
                rate(container_fs_writes_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m])
              )
            )
            * on(cluster, pod) group_left(persistentvolumeclaim, exported_namespace)
            pod_pvc_info
            * on(cluster, exported_namespace, persistentvolumeclaim) group_left(storageclass)
            pvc_storage_info

        - record: pvc_iops_utilization_percent
          expr: |
            sum by(cluster, namespace, pod, persistentvolumeclaim, storageclass) (pvc_current_iops)
            * 100
            / on(cluster, storageclass) group_left()
            max by(cluster, storageclass) (storage_class_iops_limits)
---
- alert: HighIOPSUtilization
  expr: round(pvc_iops_utilization_percent) > 85
  for: 40m
  labels:
    severity: warning
  annotations:
    summary: "IOPS is `{{ $value }}%` on `{{ $labels.cluster }}`/`{{ $labels.namespace }}`/`{{ $labels.pod }}`."

Switched terragrunt to version "0.72.3" 
time terragrunt plan
Acquiring state lock. This may take a few moments...

No changes. Your infrastructure matches the configuration.

real    0m54.200s
user    0m28.351s
sys     0m2.390s

Switched terragrunt to version "0.54.15"
time terragrunt plan
Acquiring state lock. This may take a few moments...

No changes. Your infrastructure matches the configuration.

real    0m43.498s
user    0m31.284s
sys     0m4.366s

Switched terragrunt to version "0.56.0"
time terragrunt plan
Acquiring state lock. This may take a few moments...

No changes. Your infrastructure matches the configuration.

real    3m29.356s
user    2m27.303s
sys     0m14.885s

apiclient -u /settings | jq

[settings.kubernetes]
api-server = "${endpoint}"
cluster-certificate = "${cluster_auth_base64}"
cluster-name = "${cluster_name}"
pod-pids-limit = 1024
image-gc-high-threshold-percent = 85
image-gc-low-threshold-percent = 80
container-log-max-files = 50
container-log-max-size = "100Mi"
[settings.kubernetes.system-reserved]
cpu = "200m"
ephemeral-storage = "1Gi"
memory = "1Gi"
[settings.kubernetes.kube-reserved]
memory = "893Mi"
[settings.kernel]
lockdown = "integrity"
[settings.host-containers.admin]
enabled = ${enable_admin_container}
[settings.host-containers.control]
enabled = ${enable_control_container}
[settings.kubernetes.eviction-hard]
"memory.available" = "1Gi"
"nodefs.available" = "20%"
"nodefs.inodesFree" = "5%"
"imagefs.available" = "15%"
[settings.kubernetes.eviction-soft]
"memory.available" = "10%"
[settings.kubernetes.eviction-soft-grace-period]
"memory.available" = "30s"

resource "aws_launch_template" "node-launch-template" {
...
  user_data = base64encode(templatefile("${path.module}/../../templates/bottlerocket.tpl",
    {
      "cluster_name"             = aws_eks_cluster.eks.name #обязательный параметр!!!
      "endpoint"                 = aws_eks_cluster.eks.endpoint #обязательный параметр!!!
      "cluster_auth_base64"      = aws_eks_cluster.eks.certificate_authority[0].data #обязательный параметр!!!
      "aws_region"               = var.AWS_DEFAULT_REGION #обязательный параметр!!!
      "enable_admin_container"   = false
      "enable_control_container" = true
    }
  ))
}

memory.available: "100Mi" 
nodefs.available: "10%" 
nodefs.inodesFree: "5%" 
imagefs.available: "15%"

locals {
  aws_ami_name         = "bottlerocket"
  aws_ami_architecture = "x86_64"
  aws_ami_type         = "${upper(local.aws_ami_name)}_${local.aws_ami_architecture}"
}

data "aws_ssm_parameter" "eks_ami_image_version" {
  name = "/aws/service/${local.aws_ami_name}/aws-k8s-${aws_eks_cluster.eks.version}/${local.aws_ami_architecture}/latest/image_version"
}

resource "aws_eks_node_group" "blue-green-group" {
...
  ami_type        = local.aws_ami_type
  release_version = data.aws_ssm_parameter.eks_ami_image_version.value
...

  lifecycle {
    ignore_changes        = [release_version]
...
  }

ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   10s     v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    Ready      <none>   13s     v1.30.5-eks-baa6d11

ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   5s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   10s     v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    Ready      <none>   14s     v1.30.5-eks-baa6d11

- alert: REDIS_HIGH_CPU_USAGE
  expr: aws_elasticache_cpuutilization_average > 80
  for: 5m
  labels:
    severity: critical
    emoji: 🔥
  annotations:
    summary: '`{{ $labels.cacheclusterid }}` is using {{ $value }}% of CPU. <https://****.grafana-workspace.us-east-1.amazonaws.com/d/AWSERedis/aws-elasticache-redis?var-cacheclusterid={{ $labels.cacheclusterid }}|Dashboard> | <https://****.pagerduty.com/incidents|PD>'

- alert: RDS_HIGH_CPU_USAGE
  expr: round(avg_over_time(aws_rds_cpuutilization_average[5m]), 0.01) > 80
  for: 5m
  labels:
    severity: critical
    emoji: 🔥
  annotations:
    summary: '`{{ $labels.dbinstance_identifier }}` is using {{ $value }}% of CPU. <https://****.grafana-workspace.us-east-1.amazonaws.com/d/AWSRDSdbi/aws-rds?var-dbinstanceidentifier={{ $labels.dbinstance_identifier }}|Dashboard> | <https://***.pagerduty.com/incidents|PD>'

# Alarm for Primary Node
resource "aws_cloudwatch_metric_alarm" "redis_cpu_primary" {
  alarm_name          = "${var.project_prefix}-${var.environment}-redis-cpu-primary"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = "300"
  metric_name         = "CPUUtilization"
  namespace          = "AWS/ElastiCache"
  period             = "60"
  statistic          = "Average"
  threshold          = "80"
  alarm_description  = "Redis CPU utilization is too high on primary node"
  alarm_actions      = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  ok_actions         = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  dimensions = {
    CacheClusterId = "${aws_elasticache_replication_group.******.id}-001"
  }
}

# Alarm for Replica Node
resource "aws_cloudwatch_metric_alarm" "redis_cpu_replica" {
  alarm_name          = "${var.project_prefix}-${var.environment}-redis-cpu-replica"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = "300"
  metric_name         = "CPUUtilization"
  namespace          = "AWS/ElastiCache"
  period             = "60"
  statistic          = "Average"
  threshold          = "80"
  alarm_description  = "Redis CPU utilization is too high on replica node"
  alarm_actions      = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  ok_actions         = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  dimensions = {
    CacheClusterId = "${aws_elasticache_replication_group.******.id}-002"
  }
}

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

helm upgrade cloudwatch-exporter prometheus-community/prometheus-cloudwatch-exporter \
  --install \
  --version 0.25.0 \
  --namespace monitoring \
  --set aws.aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" \
  --set aws.aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \
  --values=EKS/cloudwatch-values.yaml

config: |-
  region: us-east-1
  period_seconds: 30
  delay_seconds: 30
  metrics:
  - aws_namespace: AWS/RDS
    aws_metric_name: CPUUtilization
    aws_dimensions: [DBInstanceIdentifier]
    aws_dimension_select_regex:
      DBInstanceIdentifier: [".*production.*"]
    aws_statistics: [Average]
  - aws_namespace: AWS/SQS
    aws_metric_name: ApproximateNumberOfMessagesVisible
    aws_dimensions: [QueueName]
    aws_dimension_select_regex:
      QueueName: [".*production.*"]
    aws_statistics: [Sum]
  - aws_namespace: AWS/ES
    aws_metric_name: FreeStorageSpace
    period_seconds: 60
    aws_dimensions: [DomainName, ClientId, NodeId]
    aws_statistics: [Minimum]
  - aws_namespace: AWS/ElastiCache
    aws_metric_name: CPUUtilization
    period_seconds: 60
    aws_dimensions: [CacheClusterId]
    aws_statistics: [Average]

- job_name: 'cloudwatch-exporter'
  scrape_interval: 30s
  static_configs:
    - targets: ['cloudwatch-exporter-prometheus-cloudwatch-exporter.monitoring.svc.cluster.local:9106']