diff --git a/config/custom_templates/infrastructure_health.jinja b/config/custom_templates/infrastructure_health.jinja index 9a13e2a5..ebd5a68f 100644 --- a/config/custom_templates/infrastructure_health.jinja +++ b/config/custom_templates/infrastructure_health.jinja @@ -1,4 +1,4 @@ -{# Shared UPS interpretation and dashboard health-source inventory. #} +{# Shared UPS interpretation and dashboard health-source inventory/severity. #} {% macro eaton_low_voltage_policy(returns) -%} {% set raw = states('sensor.office_ups_eaton_550_status_data') | upper | trim %} {% set tokens = raw.split() %} @@ -36,6 +36,45 @@ }) %} {%- endmacro %} +{% macro dashboard_severity(returns) -%} + {# Match the Active Issue rows; an unrelated warning must never mask a fault. #} + {% set active = (dashboard_active_entities | as_function)() %} + {% set maintenance = [ + 'binary_sensor.node_proxmox1_updates_packages', + 'binary_sensor.node_proxmox02_updates_packages', + 'binary_sensor.docker_host_maintenance_attention', + 'binary_sensor.hvac_maintenance_due' + ] %} + {% set health_sensors = [ + 'sensor.bearclaw_scheduled_job_health', + 'sensor.joanna_onenote_kb_health' + ] %} + {% set ns = namespace(severity='critical' if states('sensor.active_issues') | int(0) > 0 else 'healthy') %} + {% for entity_id in active %} + {% set severity = 'critical' %} + {% if entity_id in maintenance and states(entity_id) == 'on' %} + {% set severity = 'warning' %} + {% elif entity_id in health_sensors and states(entity_id) == 'warning' %} + {% set severity = 'warning' %} + {% elif entity_id == 'binary_sensor.infra_wan_quality_degraded' and states(entity_id) == 'on' and + states('binary_sensor.infra_wan_critical') != 'on' %} + {% set severity = 'warning' %} + {% elif entity_id == 'binary_sensor.tesla_home_charge_attention' and states(entity_id) == 'on' and + states('binary_sensor.tesla_home_charge_critical') != 'on' %} + {% set severity = 'warning' %} + {% elif entity_id == 'binary_sensor.home_water_problem' and states(entity_id) == 'on' and + state_attr(entity_id, 'active_entities') == ['binary_sensor.rheem_wh_problem'] and + states('binary_sensor.rheem_wh_telemetry_stale') == 'on' and + states('binary_sensor.rheem_wh_active_alert') != 'on' %} + {% set severity = 'warning' %} + {% endif %} + {% if severity == 'critical' or ns.severity == 'healthy' %} + {% set ns.severity = severity %} + {% endif %} + {% endfor %} + {% do returns(ns.severity) %} +{%- endmacro %} + {% macro ups_issues(returns) -%} {% set devices = [ ('Garage UPS', 'sensor.garage_ups_status_data', 'sensor.garage_ups_status', none), diff --git a/config/dashboards/infrastructure/partials/home_sections.yaml b/config/dashboards/infrastructure/partials/home_sections.yaml index dc1a7ebc..9c67d824 100644 --- a/config/dashboards/infrastructure/partials/home_sections.yaml +++ b/config/dashboards/infrastructure/partials/home_sections.yaml @@ -18,6 +18,7 @@ # Notes: Back to Overview is the first control in the landing-view navigation strip. # Notes: UPS health covers both Garage and Office NUT servers and drills into the shared UPS view. # Notes: UPS alarms and telemetry loss roll up alongside utility outages; other submenu faults and Repairs gate Systems all-clear. +# Notes: Systems color follows the shared highest severity, with warnings amber and any critical issue red. # Notes: Missing JuiceBox MQTT reports become a critical Systems issue after # five minutes; fresh reports clear it and restore the Car view. ###################################################################### @@ -62,30 +63,9 @@ variables: ok_state: '0' healthy_text: ALL CLEAR - warning_text: > - [[[ - return states['binary_sensor.hvac_maintenance_due']?.state === 'on' - || states['binary_sensor.docker_host_maintenance_attention']?.state === 'on' - ? 'MAINTENANCE' - : 'WARNING'; - ]]] + warning_text: '[[[ return `${entity.state} ACTIVE`; ]]]' raw_suffix: ACTIVE - warning_entity_ids: - - binary_sensor.tesla_home_charge_attention - - binary_sensor.infra_wan_sustained_degradation - - binary_sensor.rheem_wh_telemetry_stale - - binary_sensor.docker_host_maintenance_attention - - binary_sensor.hvac_maintenance_due - critical_entity_ids: - - binary_sensor.infra_wan_critical - - binary_sensor.infra_dns_pihole_degraded - - binary_sensor.infra_ups_problem - - binary_sensor.infra_compute_degraded - - binary_sensor.infra_storage_degraded - - binary_sensor.infra_external_monitoring_degraded - - binary_sensor.tesla_home_charge_critical - - binary_sensor.rheem_wh_active_alert - - binary_sensor.garage_phyn_leak_alert + severity_attribute: severity - type: custom:button-card template: bearstone_infra_status_kpi diff --git a/config/dashboards/infrastructure/templates/button_card_templates.yaml b/config/dashboards/infrastructure/templates/button_card_templates.yaml index 41bf8dfd..5d72f571 100644 --- a/config/dashboards/infrastructure/templates/button_card_templates.yaml +++ b/config/dashboards/infrastructure/templates/button_card_templates.yaml @@ -964,6 +964,9 @@ bearstone_infra_status_kpi: const raw = String(entity?.state ?? '').toLowerCase(); const unavailable = ['unknown', 'unavailable', 'none', ''].includes(raw); if (unavailable) return 'unknown'; + const severityAttribute = String(variables?.severity_attribute ?? '').trim(); + const rollupSeverity = String(entity?.attributes?.[severityAttribute] ?? '').toLowerCase(); + if (['healthy', 'warning', 'critical'].includes(rollupSeverity)) return rollupSeverity; const problemEntityId = String(variables?.problem_entity_id ?? '').trim(); if (problemEntityId) { const issues = states[problemEntityId]?.attributes?.issues; @@ -1090,6 +1093,9 @@ bearstone_overview_ribbon_control: const ok = Array.isArray(variables?.ok_states) ? variables.ok_states.map(String) : []; const active = Array.isArray(variables?.active_states) ? variables.active_states.map(String) : []; const warning = Array.isArray(variables?.warning_states) ? variables.warning_states.map(String) : []; + const severityAttribute = String(variables?.severity_attribute ?? '').trim(); + const rollupSeverity = String(entity?.attributes?.[severityAttribute] ?? '').toLowerCase(); + if (['healthy', 'warning', 'critical'].includes(rollupSeverity)) return rollupSeverity; if (anyOn(normalizeIds(variables?.critical_entity_ids))) return 'critical'; if (anyOn(normalizeIds(variables?.warning_entity_ids))) return 'warning'; if (variables?.cooling_active || active.includes(raw)) return 'active'; diff --git a/config/dashboards/overview/sections/00_summary.yaml b/config/dashboards/overview/sections/00_summary.yaml index 20a429f2..52134a73 100644 --- a/config/dashboards/overview/sections/00_summary.yaml +++ b/config/dashboards/overview/sections/00_summary.yaml @@ -45,18 +45,7 @@ cards: state_labels: '0': ALL CLEAR raw_suffix: ACTIVE - warning_entity_ids: - - binary_sensor.infra_wan_sustained_degradation - - binary_sensor.rheem_wh_telemetry_stale - - binary_sensor.docker_host_maintenance_attention - - binary_sensor.hvac_maintenance_due - critical_entity_ids: - - binary_sensor.infra_ups_problem - - binary_sensor.infra_wan_critical - - binary_sensor.infra_dns_pihole_degraded - - binary_sensor.infra_compute_degraded - - binary_sensor.infra_storage_degraded - - binary_sensor.infra_external_monitoring_degraded + severity_attribute: severity tap_action: action: navigate navigation_path: /dashboard-infrastructure/home diff --git a/config/dashboards/overview/views/02_mobile.yaml b/config/dashboards/overview/views/02_mobile.yaml index f65fd051..4d7344d6 100644 --- a/config/dashboards/overview/views/02_mobile.yaml +++ b/config/dashboards/overview/views/02_mobile.yaml @@ -199,18 +199,7 @@ sections: state_labels: '0': ALL CLEAR raw_suffix: ACTIVE - warning_entity_ids: - - binary_sensor.infra_wan_sustained_degradation - - binary_sensor.rheem_wh_telemetry_stale - - binary_sensor.docker_host_maintenance_attention - - binary_sensor.hvac_maintenance_due - critical_entity_ids: - - binary_sensor.infra_ups_problem - - binary_sensor.infra_wan_critical - - binary_sensor.infra_dns_pihole_degraded - - binary_sensor.infra_compute_degraded - - binary_sensor.infra_storage_degraded - - binary_sensor.infra_external_monitoring_degraded + severity_attribute: severity tap_action: action: navigate navigation_path: /dashboard-infrastructure/home diff --git a/config/packages/README.md b/config/packages/README.md index ce5f09c8..5a052025 100755 --- a/config/packages/README.md +++ b/config/packages/README.md @@ -60,7 +60,7 @@ Kitchen voice uses Kiosk Satellite's [native Assist satellite](https://kiosksate | [![YAML source: docker_infrastructure](https://img.shields.io/static/v1?label=YAML&message=docker_infrastructure&color=lightgrey&logo=github&logoColor=181717)](docker_infrastructure.yaml) | Docker host patching telemetry with a staggered daily 06:00 check-then-update window, persisted maintenance deadlines, container/stack Repairs automation, retained Portainer health telemetry, retired Portainer image-update controls, 20-minute Joanna escalation for persistent container outages including stuck `restarting`/`created` states, and weekly scheduled prune actions across docker_10/13/14/17/69. [![Watch on YouTube](https://img.shields.io/badge/Watch-YouTube-FF0000?logo=youtube&logoColor=white)](https://youtu.be/aceV7ObKJVA) [![vCloudInfo Blog Post](https://img.shields.io/static/v1?label=vCloudInfo&message=Blog%20Post&color=21759B&logo=wordpress&logoColor=white)](https://www.vcloudinfo.com/2026/07/using-codex-clean-up-home-assistant-entities.html) | `sensor.docker_*_apt_status`, `input_datetime.apt_docker_*_action_due`, `binary_sensor.portainer_status_2`, `sensor.portainer_cpu_usage_total_2`, `sensor.portainer_memory_usage_percentage_2`, `binary_sensor.*_stack_status`, `sensor.docker_stacks_down_count`, `repairs.create`, `repairs.remove`, `script.joanna_dispatch` | | [![YAML source: proxmox](https://img.shields.io/static/v1?label=YAML&message=proxmox&color=lightgrey&logo=github&logoColor=181717)](proxmox.yaml) | Proxmox update detection with amber dashboard maintenance state, 02:15 Joanna patch orchestration, a self-releasing four-hour container-alert fail-safe lease, kernel-refresh handoff hints, sequential migration and capacity gates, plus runtime and disk pressure monitoring. Docker14 recovery startup is pre-authorized only after expected VM placement and memory headroom are verified. Update availability does not create Repairs. | `binary_sensor.node_proxmox*_updates_packages`, `sensor.node_proxmox*_total_updates`, `input_datetime.docker_container_alerts_snooze_until`, `script.joanna_dispatch`, `binary_sensor.proxmox*_runtime_healthy`, `sensor.proxmox*_disk_used_percentage` | | [![YAML source: synology_dsm](https://img.shields.io/static/v1?label=YAML&message=synology_dsm&color=lightgrey&logo=github&logoColor=181717)](synology_dsm.yaml) | Synology DSM integration health normalization for Carlo-NAS01 and Carlo-NVR, with outage-aware Joanna-first handling for lone post-outage volume warnings and Repairs escalation for persistent or non-outage problems. | `binary_sensor.carlo_*_synology_problem`, `sensor.carlo_*_synology_problem_summary`, `binary_sensor.powerwall_grid_status`, `repairs.create`, `script.joanna_dispatch` | -| [![YAML source: infrastructure](https://img.shields.io/static/v1?label=YAML&message=infrastructure&color=lightgrey&logo=github&logoColor=181717)](infrastructure.yaml) | Authenticated Cloudflare origin/tunnel/certificate alerts with persistent Repairs and bounded owned-resource Joanna diagnosis. UPS alarms, abnormal status, and telemetry loss feed the shared Systems rollup; sustained faults create persistent Repairs and deduplicated Joanna diagnosis, while verified recovery clears them. The exact Eaton low-voltage warning alone is acknowledged after a restoring 24-hour grace period on confirmed utility power without replacement, internal, or other faults; raw NUT is retained, and new outages reset the period. Systems also includes submenu health warnings and uses Repairs as an all-clear gate. Corroborated multi-probe WAN reachability, continuous latency/loss quality tiers, separate Ookla CLI capacity snapshots, normalized DNS/backup/domain/cert health, maintenance-aware Nebula Sync and promoted IoT primary/backup Pi-hole consistency monitoring with Joanna dispatch, Glances-backed Docker host disk pressure with Joanna-only warning cleanup and critical Repairs, quiet Bear Stone and website-down Repairs/Joanna remediation without phone alerts, uptime/latency SLO signals, nightly backup verification, and monthly Joanna HA log hygiene review with public-safe GitHub issue follow-up. Nebula remediation is container-scoped and never changes Docker14 VM power or placement. [![Watch on YouTube](https://img.shields.io/badge/Watch-YouTube-FF0000?logo=youtube&logoColor=white)](https://youtu.be/F-PpsKkzcrM) [![vCloudInfo Blog Post](https://img.shields.io/static/v1?label=vCloudInfo&message=Blog%20Post&color=21759B&logo=wordpress&logoColor=white)](https://www.vcloudinfo.com/2026/08/home-assistant-internet-speed-monitoring-ookla.html) | `sensor.infra_wan_reachability`, `binary_sensor.infra_wan_capacity_degraded`, `binary_sensor.infra_wan_capacity_stale`, `binary_sensor.infra_wan_sustained_degradation`, `binary_sensor.infra_wan_severe_degradation`, `binary_sensor.infra_wan_critical`, `sensor.ookla_speedtest_download`, `sensor.ookla_speedtest_upload`, `sensor.infra_nebula_sync_dns_consistency`, `sensor.infra_pihole_iot_dns_consistency`, `binary_sensor.infra_nebula_sync_degraded`, `binary_sensor.infra_pihole_iot_dns_degraded`, `sensor.docker_*_disk_used_percentage`, `automation.infra_nebula_sync_health_dispatch`, `automation.infra_pihole_iot_dns_drift_dispatch`, `automation.docker_host_disk_pressure_monitor`, `automation.infrastructure_website_down_repair_and_dispatch`, `binary_sensor.infra_website_uptime_slo_breach`, `binary_sensor.infra_website_latency_degraded`, `automation.infra_backup_nightly_verification`, `script.joanna_dispatch` | +| [![YAML source: infrastructure](https://img.shields.io/static/v1?label=YAML&message=infrastructure&color=lightgrey&logo=github&logoColor=181717)](infrastructure.yaml) | Authenticated Cloudflare origin/tunnel/certificate alerts with persistent Repairs and bounded owned-resource Joanna diagnosis. UPS alarms, abnormal status, and telemetry loss feed the shared Systems rollup; sustained faults create persistent Repairs and deduplicated Joanna diagnosis, while verified recovery clears them. The exact Eaton low-voltage warning alone is acknowledged after a restoring 24-hour grace period on confirmed utility power without replacement, internal, or other faults; raw NUT is retained, and new outages reset the period. Systems also includes submenu health warnings, uses Repairs as an all-clear gate, and shares a highest-severity rollup across desktop and mobile: warning-only issues render amber, any critical issue renders red, and no issues renders green. Corroborated multi-probe WAN reachability, continuous latency/loss quality tiers, separate Ookla CLI capacity snapshots, normalized DNS/backup/domain/cert health, maintenance-aware Nebula Sync and promoted IoT primary/backup Pi-hole consistency monitoring with Joanna dispatch and bounded DNS retries that tolerate brief sync reloads while preserving sustained-failure detection, Glances-backed Docker host disk pressure with Joanna-only warning cleanup and critical Repairs, quiet Bear Stone and website-down Repairs/Joanna remediation without phone alerts, uptime/latency SLO signals, nightly backup verification, and monthly Joanna HA log hygiene review with public-safe GitHub issue follow-up. Nebula remediation is container-scoped and never changes Docker14 VM power or placement. [![Watch on YouTube](https://img.shields.io/badge/Watch-YouTube-FF0000?logo=youtube&logoColor=white)](https://youtu.be/F-PpsKkzcrM) [![vCloudInfo Blog Post](https://img.shields.io/static/v1?label=vCloudInfo&message=Blog%20Post&color=21759B&logo=wordpress&logoColor=white)](https://www.vcloudinfo.com/2026/08/home-assistant-internet-speed-monitoring-ookla.html) | `sensor.infra_wan_reachability`, `binary_sensor.infra_wan_capacity_degraded`, `binary_sensor.infra_wan_capacity_stale`, `binary_sensor.infra_wan_sustained_degradation`, `binary_sensor.infra_wan_severe_degradation`, `binary_sensor.infra_wan_critical`, `sensor.ookla_speedtest_download`, `sensor.ookla_speedtest_upload`, `sensor.infra_nebula_sync_dns_consistency`, `sensor.infra_pihole_iot_dns_consistency`, `binary_sensor.infra_nebula_sync_degraded`, `binary_sensor.infra_pihole_iot_dns_degraded`, `sensor.docker_*_disk_used_percentage`, `automation.infra_nebula_sync_health_dispatch`, `automation.infra_pihole_iot_dns_drift_dispatch`, `automation.docker_host_disk_pressure_monitor`, `automation.infrastructure_website_down_repair_and_dispatch`, `binary_sensor.infra_website_uptime_slo_breach`, `binary_sensor.infra_website_latency_degraded`, `automation.infra_backup_nightly_verification`, `script.joanna_dispatch` | | [![YAML source: bgw_gateway](https://img.shields.io/static/v1?label=YAML&message=bgw_gateway&color=lightgrey&logo=github&logoColor=181717)](bgw_gateway.yaml) | Pulls the public-safe [BGW Probe](https://github.com/CCOSTAN/bgw-probe) API into Home Assistant as gateway reachability, broadband link, uptime, negotiated speed, byte, error, and drop telemetry. [![Watch on YouTube](https://img.shields.io/badge/Watch-YouTube-FF0000?logo=youtube&logoColor=white)](https://youtu.be/ovNJnaVPWAY) [![vCloudInfo Blog Post](https://img.shields.io/static/v1?label=vCloudInfo&message=Blog%20Post&color=21759B&logo=wordpress&logoColor=white)](https://www.vcloudinfo.com/2026/09/monitor-att-bgw210-bgw320-gateway-home-assistant.html) | `sensor.bgw_gateway_status`, `binary_sensor.bgw_gateway_reachable`, `binary_sensor.bgw_broadband_connection`, `sensor.bgw_gateway_uptime`, `sensor.bgw_broadband_receive_bytes`, `sensor.bgw_broadband_transmit_bytes` | | [![YAML source: speedtest](https://img.shields.io/static/v1?label=YAML&message=speedtest&color=lightgrey&logo=github&logoColor=181717)](speedtest.yaml) | Logs corroborated WAN outages, sustained latency/loss degradation, slow Ookla capacity advisories, and recovery without letting one throughput snapshot declare the WAN down. [![Watch on YouTube](https://img.shields.io/badge/Watch-YouTube-FF0000?logo=youtube&logoColor=white)](https://youtu.be/F-PpsKkzcrM) [![vCloudInfo Blog Post](https://img.shields.io/static/v1?label=vCloudInfo&message=Blog%20Post&color=21759B&logo=wordpress&logoColor=white)](https://www.vcloudinfo.com/2026/08/home-assistant-internet-speed-monitoring-ookla.html) | `binary_sensor.infra_wan_quality_degraded`, `binary_sensor.infra_wan_outage`, `binary_sensor.infra_wan_capacity_degraded`, `sensor.ookla_speedtest_download`, `sensor.ookla_speedtest_upload`, `script.send_to_logbook` | | [![YAML source: onenote_indexer](https://img.shields.io/static/v1?label=YAML&message=onenote_indexer&color=lightgrey&logo=github&logoColor=181717)](onenote_indexer.yaml) | Dedicated-appliance OneNote indexer health/status monitoring for Joanna, explicit index-health confirmation, and failure-repair automation. Native indexer reconciliation remains the maintenance owner. | `sensor.onenote_indexer_last_job_status`, `binary_sensor.onenote_indexer_last_job_successful`, `binary_sensor.onenote_indexer_index_healthy` | diff --git a/config/packages/infrastructure.yaml b/config/packages/infrastructure.yaml index 536bfe10..1cb587db 100644 --- a/config/packages/infrastructure.yaml +++ b/config/packages/infrastructure.yaml @@ -12,9 +12,10 @@ # Related Issue: 1584 # Related Issue: 1890 # Notes: Home dashboard consumes `infra_*` entities for exceptions-only alerts. +# Notes: Systems uses the highest active severity: critical overrides warning; no issues is healthy. # Notes: Cloudflare alerts use an authenticated Nabu Casa webhook, persistent Repairs, and owned-resource-only Joanna dispatch. # Notes: Certificate and unowned-domain alerts require human review; no Cloudflare DNS or security writes are authorized by an alert. -# Notes: Five-minute read-only Cloudflare notification-history polling provides notification-only coverage when webhook destinations are unavailable. +# Notes: Cloudflare delivery is webhook-only; origin recovery reuses reports from existing website probes. # Notes: WAN outages require all four independent ICMP/HTTPS probes to fail continuously for 2 minutes. # Notes: Continuous latency/loss determine WAN quality; Ookla throughput is a separate twice-daily capacity snapshot and never declares an outage. # Notes: Severe quality turns critical after 30 minutes above 250 ms or 25% loss; moderate quality degradation requires 6 hours above 80 ms or 5% loss. @@ -29,6 +30,7 @@ # Notes: Warning-level Docker host disk pressure is Joanna-only; Repairs are reserved for critical pressure. # Notes: Nebula Sync compares the docker_10 primary with the physical office-pi DNS replica and dispatches Joanna on sustained drift or endpoint loss. # Notes: Promoted IoT DNS consistency compares primary/backup Pi-hole answers for reserved IoT host records. +# Notes: DNS probes retry brief resolver interruptions and filter transport diagnostics; timeout budgets also cover persistent resolver loss. # Notes: Garage compute health uses the same 145 F threshold as the existing temperature notification. # Notes: Immediate website-down states create Repairs; the installed UptimeRobot integration exposes current state, not trend rollups. # Notes: Bear Stone and website outages dispatch Joanna quietly; no direct phone alerts are sent. @@ -138,8 +140,9 @@ command_line: - sensor: name: Infra Nebula Sync DNS Consistency unique_id: infra_nebula_sync_dns_consistency + command_timeout: 40 command: >- - /bin/bash -c 'primary=192.168.10.10; secondary=192.168.10.100; host=GTG-PF45FK6F; fqdn=GTG-PF45FK6F.fordst.com; ip=192.168.10.117; q(){ dig +time=2 +tries=1 +short @"$1" "$2" A 2>/dev/null | tr -d "\r" | sort | tr "\n" "," | sed "s/,$//"; }; r(){ dig +time=2 +tries=1 +short @"$1" -x "$2" 2>/dev/null | tr -d "\r" | sed "s/\.$//" | sort | tr "\n" "," | sed "s/,$//"; }; p_short=$(q "$primary" "$host"); s_short=$(q "$secondary" "$host"); p_fqdn=$(q "$primary" "$fqdn"); s_fqdn=$(q "$secondary" "$fqdn"); p_rev=$(r "$primary" "$ip"); s_rev=$(r "$secondary" "$ip"); status=mismatch; if [ "$p_short" = "$ip" ] && [ "$s_short" = "$ip" ] && [ "$p_fqdn" = "$ip" ] && [ "$s_fqdn" = "$ip" ] && [ -n "$p_rev" ] && [ "$p_rev" = "$s_rev" ]; then status=ok; fi; printf "{\"status\":\"%s\",\"host\":\"%s\",\"expected_ip\":\"%s\",\"primary_short\":\"%s\",\"secondary_short\":\"%s\",\"primary_fqdn\":\"%s\",\"secondary_fqdn\":\"%s\",\"primary_reverse\":\"%s\",\"secondary_reverse\":\"%s\"}\n" "$status" "$host" "$ip" "$p_short" "$s_short" "$p_fqdn" "$s_fqdn" "$p_rev" "$s_rev"' + /bin/bash -c 'primary=192.168.10.10; secondary=192.168.10.100; host=GTG-PF45FK6F; fqdn=GTG-PF45FK6F.fordst.com; ip=192.168.10.117; q(){ dig +time=2 +tries=3 +short @"$1" "$2" A 2>/dev/null | sed "/^;/d" | tr -d "\r" | sort | tr "\n" "," | sed "s/,$//"; }; r(){ dig +time=2 +tries=3 +short @"$1" -x "$2" 2>/dev/null | sed "/^;/d" | tr -d "\r" | sed "s/\.$//" | sort | tr "\n" "," | sed "s/,$//"; }; p_short=$(q "$primary" "$host"); s_short=$(q "$secondary" "$host"); p_fqdn=$(q "$primary" "$fqdn"); s_fqdn=$(q "$secondary" "$fqdn"); p_rev=$(r "$primary" "$ip"); s_rev=$(r "$secondary" "$ip"); status=mismatch; if [ "$p_short" = "$ip" ] && [ "$s_short" = "$ip" ] && [ "$p_fqdn" = "$ip" ] && [ "$s_fqdn" = "$ip" ] && [ -n "$p_rev" ] && [ "$p_rev" = "$s_rev" ]; then status=ok; fi; printf "{\"status\":\"%s\",\"host\":\"%s\",\"expected_ip\":\"%s\",\"primary_short\":\"%s\",\"secondary_short\":\"%s\",\"primary_fqdn\":\"%s\",\"secondary_fqdn\":\"%s\",\"primary_reverse\":\"%s\",\"secondary_reverse\":\"%s\"}\n" "$status" "$host" "$ip" "$p_short" "$s_short" "$p_fqdn" "$s_fqdn" "$p_rev" "$s_rev"' scan_interval: 300 value_template: "{{ value_json.status | default('unknown') }}" json_attributes: @@ -155,8 +158,9 @@ command_line: - sensor: name: Infra Pihole IoT DNS Consistency unique_id: infra_pihole_iot_dns_consistency + command_timeout: 90 command: >- - /bin/bash -c 'primary=192.168.10.10; secondary=192.168.10.100; records="rachio.fordst.com=192.168.10.73 econet.fordst.com=192.168.10.92 dreame-vacuum.fordst.com=192.168.10.93 carlo-bed.fordst.com=192.168.10.95 lg-smart-fridge.fordst.com=192.168.10.96 tesla-blackbox-gw.fordst.com=192.168.10.97 bgw210.fordst.com=192.168.10.98"; q(){ dig +time=2 +tries=1 +short @"$1" "$2" A 2>/dev/null | tr -d "\r" | sort | tr "\n" "," | sed "s/,$//"; }; status=ok; checked=0; mismatch_count=0; mismatches=""; for record in $records; do host=${record%%=*}; ip=${record#*=}; p=$(q "$primary" "$host"); s=$(q "$secondary" "$host"); checked=$((checked+1)); if [ "$p" != "$ip" ] || [ "$s" != "$ip" ]; then status=mismatch; mismatch_count=$((mismatch_count+1)); mismatches="${mismatches}${host}:expected=${ip},primary=${p:-none},secondary=${s:-none};"; fi; done; if [ -z "$mismatches" ]; then mismatches=none; fi; printf "{\"status\":\"%s\",\"checked_records\":%s,\"mismatch_count\":%s,\"mismatches\":\"%s\",\"primary_dns\":\"%s\",\"backup_dns\":\"%s\"}\n" "$status" "$checked" "$mismatch_count" "$mismatches" "$primary" "$secondary"' + /bin/bash -c 'primary=192.168.10.10; secondary=192.168.10.100; records="rachio.fordst.com=192.168.10.73 econet.fordst.com=192.168.10.92 dreame-vacuum.fordst.com=192.168.10.93 carlo-bed.fordst.com=192.168.10.95 lg-smart-fridge.fordst.com=192.168.10.96 tesla-blackbox-gw.fordst.com=192.168.10.97 bgw210.fordst.com=192.168.10.98"; q(){ dig +time=2 +tries=3 +short @"$1" "$2" A 2>/dev/null | sed "/^;/d" | tr -d "\r" | sort | tr "\n" "," | sed "s/,$//"; }; status=ok; checked=0; mismatch_count=0; mismatches=""; for record in $records; do host=${record%%=*}; ip=${record#*=}; p=$(q "$primary" "$host"); s=$(q "$secondary" "$host"); checked=$((checked+1)); if [ "$p" != "$ip" ] || [ "$s" != "$ip" ]; then status=mismatch; mismatch_count=$((mismatch_count+1)); mismatches="${mismatches}${host}:expected=${ip},primary=${p:-none},secondary=${s:-none};"; fi; done; if [ -z "$mismatches" ]; then mismatches=none; fi; printf "{\"status\":\"%s\",\"checked_records\":%s,\"mismatch_count\":%s,\"mismatches\":\"%s\",\"primary_dns\":\"%s\",\"backup_dns\":\"%s\"}\n" "$status" "$checked" "$mismatch_count" "$mismatches" "$primary" "$secondary"' scan_interval: 300 value_template: "{{ value_json.status | default('unknown') }}" json_attributes: @@ -282,6 +286,9 @@ template: {% set active = dashboard_active_entities | as_function %} {{ active() }} repair_count: "{{ states('sensor.active_issues') | int(0) }}" + severity: >- + {% from 'infrastructure_health.jinja' import dashboard_severity %} + {{ (dashboard_severity | as_function)() }} count_policy: "Maximum of active health sources and Repairs; overlapping Repairs are not added twice." - binary_sensor: @@ -1569,7 +1576,6 @@ cloudflare_alerts: webhook_id: !secret cloudflare_alert_webhook_id secret: !secret cloudflare_alert_webhook_secret account_id: !secret cloudflare_alert_account_id - api_token: !secret cloudflare_alert_read_token ops_hostnames: !secret cloudflare_alert_ops_hostnames ops_tunnel_ids: !secret cloudflare_alert_ops_tunnel_ids recovery_entities: !secret cloudflare_alert_recovery_entities diff --git a/tools/test_infrastructure_health.py b/tools/test_infrastructure_health.py index 7666366b..351ead26 100644 --- a/tools/test_infrastructure_health.py +++ b/tools/test_infrastructure_health.py @@ -51,6 +51,8 @@ class InfrastructureHealthTest(unittest.TestCase): extensions=["jinja2.ext.do"]) self.env.filters["as_function"] = as_function self.env.globals["states"] = lambda entity_id: self.states.get(entity_id, "unknown") + self.attributes = {} + self.env.globals["state_attr"] = lambda entity_id, name: self.attributes.get(entity_id, {}).get(name) self.clock = datetime(2026, 10, 1, 21, tzinfo=timezone.utc) self.env.globals["now"] = lambda: self.clock self.env.globals["as_timestamp"] = self.timestamp @@ -229,6 +231,72 @@ class InfrastructureHealthTest(unittest.TestCase): self.states["sensor.office_ups_eaton_550_status_data"] = "ALARM OL" self.assertEqual(self.count(), 2) + def severity(self): + return self.env.from_string(self.counter["attributes"]["severity"]).render().strip() + + def test_warning_only_rollup_stays_warning_and_counts_all_sources(self): + self.assertEqual(self.severity(), "healthy") + self.states.update({ + "sensor.bearclaw_scheduled_job_health": "warning", + "sensor.joanna_onenote_kb_health": "warning", + "binary_sensor.node_proxmox1_updates_packages": "on", + }) + self.assertEqual(self.count(), 3) + self.assertEqual(self.severity(), "warning") + + def test_critical_overrides_warning_then_recovers_through_warning_to_clear(self): + self.states["sensor.bearclaw_scheduled_job_health"] = "warning" + self.states["binary_sensor.infra_nebula_sync_degraded"] = "on" + self.assertEqual(self.count(), 2) + self.assertEqual(self.severity(), "critical") + self.states["binary_sensor.infra_nebula_sync_degraded"] = "off" + self.assertEqual(self.count(), 1) + self.assertEqual(self.severity(), "warning") + self.states["sensor.bearclaw_scheduled_job_health"] = "error" + self.assertEqual(self.severity(), "critical") + self.states["sensor.bearclaw_scheduled_job_health"] = "ok" + self.assertEqual(self.count(), 0) + self.assertEqual(self.severity(), "healthy") + + def test_category_escalation_changes_severity_without_changing_issue_count(self): + for attention, critical in [ + ("binary_sensor.infra_wan_quality_degraded", "binary_sensor.infra_wan_critical"), + ("binary_sensor.tesla_home_charge_attention", "binary_sensor.tesla_home_charge_critical"), + ]: + with self.subTest(attention=attention): + self.states[attention] = "on" + self.states[critical] = "off" + self.assertEqual(self.count(), 1) + self.assertEqual(self.severity(), "warning") + self.states[critical] = "on" + self.assertEqual(self.count(), 1) + self.assertEqual(self.severity(), "critical") + self.states[attention] = "off" + self.states[critical] = "off" + + def test_water_telemetry_warning_cannot_mask_a_water_fault(self): + self.states.update({ + "binary_sensor.home_water_problem": "on", + "binary_sensor.rheem_wh_telemetry_stale": "on", + "binary_sensor.rheem_wh_active_alert": "off", + }) + self.attributes["binary_sensor.home_water_problem"] = { + "active_entities": ["binary_sensor.rheem_wh_problem"]} + self.assertEqual(self.severity(), "warning") + self.attributes["binary_sensor.home_water_problem"]["active_entities"].append( + "binary_sensor.garage_phyn_leak_alert") + self.assertEqual(self.severity(), "critical") + + def test_open_repairs_and_missing_health_never_become_warning_only_or_clear(self): + self.states["sensor.bearclaw_scheduled_job_health"] = "warning" + self.states["sensor.active_issues"] = "2" + self.assertEqual(self.count(), 2) + self.assertEqual(self.severity(), "critical") + self.states["sensor.active_issues"] = "0" + self.states["sensor.bearclaw_scheduled_job_health"] = "unavailable" + self.assertEqual(self.count(), 1) + self.assertEqual(self.severity(), "critical") + if __name__ == "__main__": unittest.main()