Hosts
- Host memory usage exceeds threshold
1 - node_memory_MemAvailable_bytes{mode!="idle"} / node_memory_MemTotal_bytes
Threshold: 0.9
- Host CPU usage exceeds threshold
1 - avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (host_name)
Threshold: 0.85
- Host disk usage exceeds threshold
1 - avg without (fstype)(node_filesystem_free_bytes{fstype!='rootfs',mountpoint!~'/(run|var|snap).*'} / node_filesystem_size_bytes{fstype!='rootfs',mountpoint!~'/(run|var|snap).*'})
Threshold: 0.8
Windows
- Windows host memory usage exceeds threshold
1 - 1 * windows_os_physical_memory_free_bytes{job="windows_exporter",mode!="idle"} / windows_cs_physical_memory_bytes
Threshold: 0.9
- Windows host CPU usage exceeds threshold
1 - (avg by (host_ip,host_name) (irate(windows_cpu_time_total{job="windows_exporter",mode="idle"}[1m])))
Threshold: 0.85
- Windows host disk usage exceeds threshold
1 - windows_logical_disk_free_bytes{job="windows_exporter",volume!~"HarddiskVolume.*"} / windows_logical_disk_size_bytes
Threshold: 0.8
Kubernetes
- Cluster component status
sum by (cluster, exported_pod, phase) (kube_pod_status_phase{phase!="Running", exported_namespace="kube-system", exported_pod=~"kube-apiserver-.*|kube-controller-manager-.*|kube-proxy-.*"} == 1) > 0
- Memory growing rapidly within 10 minutes
increase(sum(container_memory_working_set_bytes{container!~"POD", image!="",name=~"^k8s_.*"}) by (pod, namespace, cluster)) [10m] / sum(container_memory_working_set_bytes{container!~"POD", image!="",name=~"^k8s_.*"}) by (pod, namespace, cluster) offset 10m
Threshold: greater than 1
- Large-memory application detection
sum(container_memory_working_set_bytes{container!~"POD", image!="",name=~"^k8s_.*"}) by (pod, namespace, cluster)/1024/1024/1024
Threshold: greater than 2, in GB
- Number of cores throttled for a Pod’s CPU
sum (rate (container_cpu_cfs_throttled_seconds_total{name=~"^k8s_.*"}[5m])) by (pod)
Threshold: greater than 1
- Ratio of a Pod’s CPU being throttled
sum by (cluster, namespace, pod)(irate(container_cpu_cfs_throttled_periods_total{container!="POD", container!=""}[5m])) / sum by (cluster, namespace, pod)( irate(container_cpu_cfs_periods_total{container!="POD", container!=""}[5m])) > 0.5
Threshold: greater than 0.5
Another way to write it is
sum by (cluster, namespace, pod) (irate (container_cpu_cfs_throttled_seconds_total{name=~"^k8s_.*"}[10s])) / sum by (cluster, namespace, pod) (kube_pod_container_resource_limits{resource="cpu"}) > 0.5
- POD restarts more than 3 times in 10 minutes
sum (increase (kube_pod_container_status_restarts_total{}[10m])) by (cluster, namespace,pod)
Threshold: greater than 3
- POD OOM within 10 minutes
sum by (namespace,pod) ((kube_pod_container_status_restarts_total{} - kube_pod_container_status_restarts_total{} offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason='OOMKilled'}[10m]) == 1)
Threshold: greater than 2
- POD waiting
sum by (namespace, pod, reason) (kube_pod_container_status_waiting_reason{})
Threshold: greater than 0
- POD Pending
sum by (namespace, pod)(kube_pod_status_ready{condition='false'})
Threshold: greater than 0
- Cluster count changes
count(count(kube_node_created{}) by (cluster))
Threshold: not equal to the real value
- Increase in unavailable nodes over 5 minutes
sum(kube_node_status_condition{status!="true"}) by (cluster, exported_node, condition) - sum(kube_node_status_condition{status!="true"} offset 5m) by (cluster, exported_node, condition)
Threshold: greater than 0
- Node clock out of sync
min_over_time(node_timex_sync_status[5m])
Threshold: equal to 0
- Controller scheduling depth
sum by (cluster) (workqueue_depth{})
Threshold: greater than 0
- Total queue additions over 5 minutes
sum by (cluster) (rate(workqueue_adds_total{}[5m]))
Threshold: greater than 50
- Time an object spends in the queue
histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{}[5m])) by (cluster, le))
Threshold: greater than 0.5
ClickHouse
- Replica sync queue is backing up
clickhouse_replicas_max_queue_size{}
Threshold: greater than 100
Elasticsearch
- Elastic_Cluster_Health_RED: some primary shards are not running normally
elasticsearch_cluster_health_status{color="red"}
Threshold: equal to 1
- Elasticsearch_health_up: the cluster is in an unhealthy state
elasticsearch_cluster_health_up{}
Threshold: not equal to 1
- ES heap memory usage above 90%
sum by ( cluster_name, host) (elasticsearch_jvm_memory_used_bytes{area="heap"}) / sum by ( cluster_name, host) (elasticsearch_jvm_memory_max_bytes{area="heap"})
Threshold: greater than 0.9
- ES number of pending tasks greater than 10
elasticsearch_cluster_health_number_of_pending_tasks{}
Threshold: greater than 10
MongoDB
- MongoDB ops commands too high
mongodb_commands_per_sec{} or cloudwatch_aws_doc_db_opcounters_command_average{}
Threshold: greater than 5000
- MongoDB connection count too high
mongodb_open_connections{} or cloudwatch_aws_doc_db_database_connections_average{}
Threshold: greater than 5000
- MongoDB wiredTiger cache reaches the eviction threshold
sum by (cluster_name) (mongodb_wtcache_current_bytes{}) / sum by (cluster_name) (mongodb_wtcache_max_bytes_configured{}) * 100
Threshold: greater than 90
- MongoDB wiredTiger dirty cache reaches the eviction threshold
sum by (cluster_name) (mongodb_wtcache_tracked_dirty_bytes{}) / sum by (cluster_name) (mongodb_wtcache_max_bytes_configured{}) * 100
Threshold: greater than 20
Redis
- Service status abnormal
redis_up{job="telegraf"}
Threshold: less than 1
- Current instance memory usage too high
redis_memory_used_bytes{job="telegraf"} / redis_config_maxmemory
Threshold: greater than 0.8
- Instance memory usage grows more than 50% in 10 minutes
(redis_memory_used_bytes{job="telegraf"} / redis_config_maxmemory) - (redis_memory_used_bytes{job="telegraf"} offset 10m / redis_config_maxmemory offset 10m)
Threshold: greater than 0.5
- Current cluster memory usage too high
sum(redis_memory_used_bytes{job="telegraf"}) by (cluster_name) / sum(redis_config_maxmemory{job="telegraf"}) by (cluster_name)
Threshold: greater than 0.8
- Cluster memory usage grows more than 10% in 10 minutes
(sum(redis_memory_used_bytes{job="telegraf"}) by (cluster_name) / sum(redis_config_maxmemory{job="telegraf"}) by (cluster_name))- (sum(redis_memory_used_bytes{job="telegraf"}offset 10m) by (cluster_name) / sum(redis_config_maxmemory{job="telegraf"}offset 10m) by (cluster_name))
Threshold: greater than 0.1
- QPS grows more than 30% in 10 minutes and current QPS > 500
sum by (cluster_name) (rate(redis_commands_processed_total{job="telegraf"}[5m])) and ((sum by (cluster_name) (rate(redis_commands_processed_total{job="telegraf"}[5m] )) -sum by (cluster_name) (rate(redis_commands_processed_total{job="telegraf"}[5m] offset 10m)) ) /sum by (cluster_name) (rate(redis_commands_processed_total{job="telegraf""}[5m] offset 10m))> 0.3)
Threshold: greater than 500
- Connection count grows more than 30% in 10 minutes and current connection count > 500
sum by (cluster_name) (redis_connected_clients{job="telegraf"}) and (((sum by (cluster_name) (redis_connected_clients{job="telegraf"}) -sum by (cluster_name) (redis_connected_clients{job="telegraf"}offset 10m))) / sum by (cluster_name) (redis_connected_clients{job="telegraf"}offset 10m)> 0.3)
Threshold: > 500
RabbitMQ
- Instance liveness probe abnormal
avg by (cluster_name,addr) (rabbitmq_up{})
Threshold: < 1
- Memory usage too high
sum by (cluster_name,region,addr) (rabbitmq_node_mem_used{}) / sum by (cluster_name,region,addr) (rabbitmq_node_mem_limit{})
Threshold: > 0.85
- Connection usage too high
sum by (cluster_name,region,addr) (rabbitmq_sockets_used{}) / sum by (cluster_name,region,addr) (rabbitmq_sockets_available{})
Threshold: > 0.85
Pika
- Pika node ops too high
sum by(cluster_name, addr ) (rate(pika_total_commands_processed[5m]))
Threshold: > 120000
- Pika node status abnormal
avg(pika_up{}) by (cluster_name,addr)
Threshold: < 1
- Pika slave lag too large
avg(pika_slave_lag{}) by (cluster_name,addr)
Threshold: >100000
- Pika master connection abnormal
avg(pika_master_link_up{}) by (cluster_name,addr)
Threshold: <1
MySQL
- Disk usage too high
(huaweicloud_sys_rds_rds048_disk_used_size / huaweicloud_sys_rds_rds047_disk_total_size) or (1 - aws_rdsdisk_free / aws_rdsdisk_total)
Threshold: > 0.8
- CPU usage too high
huaweicloud_sys_rds_rds001_cpu_util or aws_rds_cpuutilization_average
Threshold: > 80
- Too many slow queries
rate(mysql_slow_queries{}[5m])
Threshold: > 50
- Connection count too high
mysql_threads_connected{}
Threshold: > 2000
- Connection usage too high
mysql_threads_connected{} / mysql_max_used_connections{}
Threshold: > 0.9
- MySQL QPS too high
rate(mysql_queries{}[5m])
Threshold: > 8000
- MySQL liveness probe failed
mysql_up{job="telegraf-mysql"}
Threshold: < 1
- MySQL disk usage too high
((huaweicloud_sys_rds_rds048_disk_used_size / huaweicloud_sys_rds_rds047_disk_total_size) and on (name) mysql_up{}) or ((1 - aws_rdsdisk_free / aws_rdsdisk_total) and on (name) mysql_up{})
Threshold > 0.8
- MySQL CPU usage too high
(huaweicloud_sys_rds_rds001_cpu_util and on (name) mysql_up{}) or (aws_rds_cpuutilization_average and on (dbinstance_identifier) mysql_up{})
Threshold: > 80
Kafka
- Kafka queue backlog too large
avg without(host_ip) (sum by(consumergroup, topic, cluster_name, host_ip) (kafka_consumergroup_lag{})) > 1000 and on (cluster_name, consumergroup, topic ) sum without(partition, job, host_ip) (delta(kafka_consumergroup_current_offset{}[1d]))
Threshold: != 0
- Kafka has under-replicated partitions
avg_over_time(kafka_server_replicamanager_underreplicatedpartitions{}[1m])
Threshold: >=1
- Kafka has partitions without a leader
avg_over_time(kafka_controller_kafkacontroller_offlinepartitionscount{}[1m])
Threshold: > 0
- The number of controllers in the Kafka cluster equals 0
sum by(cluster_name) (avg_over_time(kafka_controller_kafkacontroller_activecontrollercount{}[2m]))
Threshold: =0
- Insufficient number of Kafka brokers
kafka_brokers{}
Threshold: =0
- Zookeeper instance is down
zk_up{}
Threshold: =0
- Zookeeper connection count too high
zk_num_alive_connections{}
Threshold: > 100
- Zookeeper number of queued requests beyond server capacity
zk_outstanding_requests{}
Threshold: >100
- Zookeeper average time spent responding to client requests too long (ms)
zk_avg_latency{}
Threshold: >100
Etcd
- Etcd Server has no leader
max by (cluster) (etcd_server_has_leader{}) == 0
Threshold: = 0
- Etcd proposls failed num > 10 within 2min
increase(etcd_server_proposals_failed_total{}[2m])
Threshold: >10
- Etcd Disk fsync durations > 500ms
histogram_quantile(0.99, sum by(instance,le,cluster_name) (rate(etcd_disk_wal_fsync_duration_seconds_bucket{}[2m])))
Threshold: > 0.5
- Etcd Disk commit durations > 250ms
histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{}[2m]))
Threshold: > 0.25
- Etcd leader changes num > 2 within 5min
increase(etcd_server_leader_changes_seen_total{}[5m])
Threshold: > 2
- Etcd DB space usage over 85%
etcd_mvcc_db_total_size_in_bytes{} / etcd_server_quota_backend_bytes{}
Threshold: > 0.85
