ob_rules.yaml

groups:
  - name: ob-alert
    rules:
      - alert: ob_host_connection_percent_over_threshold
        expr: 100 * ob_active_session_num / 262144 > 80
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} connection used percent over threshold "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} connection_used_percent = {{ $value }}"

      - alert: ob_cluster_frozen_version_delta_over_threshold
        expr: max(ob_zone_stat{name="frozen_version"}) by (ob_cluster_name, ob_cluster_id) - min(ob_zone_stat{name="last_merged_version"}) by (ob_cluster_name, ob_cluster_id) > 1
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} too much frozen memstore not merged "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} no_merge_memstore_count = {{ $value }}"

      - alert: ob_cluster_exists_inactive_server
        expr: max(ob_server_num{status="inactive"}) by (ob_cluster_name, ob_cluster_id) > 0
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} exists inactive observer "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} inactive observer count is {{ $value }}"

      - alert: ob_cluster_exists_index_fail_table
        expr: sum(ob_index_error_num) by (ob_cluster_name, ob_cluster_id) > 0
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} exists error index table"
          description: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} error index table count is {{ $value }}"

      - alert: ob_cluster_merge_timeout
        expr: max(ob_zone_stat{name="is_merge_timeout"}) by (ob_cluster_name, ob_cluster_id) == 1
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} merge time out"
          description: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} merge time out"

      - alert: ob_cluster_merge_error
        expr: max(ob_zone_stat{name="is_merge_error"}) by (ob_cluster_name, ob_cluster_id) == 1
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} merge error"
          description: "{{ $labels.ob_cluster_name }} {{ $labels.ob_cluster_id }} merge error"

      - alert: ob_host_partition_count_over_threshold
        expr: ob_partition_num > 30000
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} partition count over threshold "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} partition_count = {{ $value }}"

      - alert: ob_server_sstable_percent_over_threshold
        expr: 100 * (ob_disk_total_bytes - ob_disk_free_bytes) / ob_disk_total_bytes > 85
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} sstable used percent over threshold "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} sstable used percent = {{ $value }}"

      - alert: tenant_cpu_percent_over_threshold
        expr: 100  * sum(ob_sysstat{stat_id="140006"}) by (svr_ip, tenant_name) / sum(ob_sysstat{stat_id="140005"}) by (svr_ip, tenant_name) > 90
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} {{ $labels.tenant_name }} cpu used percent over threshold "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} {{ $labels.tenant_name }} cpu used percent = {{ $value }}"

      - alert: tenant_active_memstore_percent_over_threshold
        expr: 100 * sum(ob_sysstat{stat_id="130000"}) by (svr_ip, tenant_name) / sum(ob_sysstat{stat_id="130002"}) by (svr_ip, tenant_name) > 110
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} {{ $labels.tenant_name }} tenant active memstore used percent over threshold "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} {{ $labels.tenant_name }} tenant active memstore used percent = {{ $value }}"

      - alert: tenant_memstore_percent_over_threshold
        expr: 100 * sum(ob_sysstat{stat_id="130001"}) by (svr_ip, tenant_name) / sum(ob_sysstat{stat_id="130004"}) by (svr_ip, tenant_name) > 85
        for: 1m
        labels:
          serverity: page
        annotations:
          summary: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} {{ $labels.tenant_name }} tenant memstore used percent over threshold "
          description: "{{ $labels.ob_cluster_name }} {{ $labels.obzone }} {{ $labels.svr_ip }} {{ $labels.tenant_name }} tenant memstore used percent = {{ $value }}"