From 2240a554dcd82124c3bb8b458a53410780b6439a Mon Sep 17 00:00:00 2001 From: Carlo Costanzo Date: Tue, 14 Apr 2026 12:25:29 -0400 Subject: [PATCH] Add nightly Duplicati verification via BearClaw --- .../partials/home_sections.yaml | 35 ------ config/packages/bearclaw.yaml | 13 +++ .../infrastructure_observability.yaml | 110 +++++++++++++----- config/recorder.yaml | 3 - config/travis_secrets.yaml | 1 + 5 files changed, 98 insertions(+), 64 deletions(-) diff --git a/config/dashboards/infrastructure/partials/home_sections.yaml b/config/dashboards/infrastructure/partials/home_sections.yaml index 2c67b332..f5222053 100644 --- a/config/dashboards/infrastructure/partials/home_sections.yaml +++ b/config/dashboards/infrastructure/partials/home_sections.yaml @@ -372,41 +372,6 @@ state_display: > [[[ return `${entity.state}% used`; ]]] - - type: custom:button-card - template: bearstone_infra_list_row - name: Backup stale or failed - icon: mdi:backup-restore - state_display: > - [[[ - const s = states['sensor.dockerconfigs_backup_status']?.state ?? 'unknown'; - const e = states['sensor.dockerconfigs_backup_error_message']?.state ?? ''; - const d = states['sensor.dockerconfigs_backup_date']?.state; - let ageText = 'n/a'; - if (d && !['unknown','unavailable','none',''].includes(String(d).toLowerCase())) { - const dt = new Date(d); - if (!Number.isNaN(dt.getTime())) { - const h = (Date.now() - dt.getTime()) / 3600000; - ageText = `${h.toFixed(1)}h`; - } - } - return `Age ${ageText} | ${s}${e ? ' | error' : ''}`; - ]]] - styles: - card: - - display: > - [[[ - const status = String(states['sensor.dockerconfigs_backup_status']?.state ?? '').toLowerCase(); - const err = String(states['sensor.dockerconfigs_backup_error_message']?.state ?? '').toLowerCase(); - const d = states['sensor.dockerconfigs_backup_date']?.state; - let stale = false; - if (d && !['unknown','unavailable','none',''].includes(String(d).toLowerCase())) { - const dt = new Date(d); - if (!Number.isNaN(dt.getTime())) stale = ((Date.now() - dt.getTime()) / 3600000) > 24; - } - const failed = status.includes('fail') || status.includes('error') || err.length > 0; - return (stale || failed) ? 'block' : 'none'; - ]]] - - type: custom:button-card template: bearstone_infra_list_row name: Services diff --git a/config/packages/bearclaw.yaml b/config/packages/bearclaw.yaml index 98945847..068e8094 100644 --- a/config/packages/bearclaw.yaml +++ b/config/packages/bearclaw.yaml @@ -14,6 +14,7 @@ # Notes: Inbound Telegram handling enforces user_id + chat_id allowlists from secrets CSV values. # Notes: Reply webhook writes JOANNA activity entries to logbook for traceability. # Notes: Status telemetry polling expects !secret bearclaw_status_url (token header stays !secret bearclaw_token). +# Notes: Nightly Duplicati verification calls a codex_appliance admin endpoint and returns structured health to HA via response_variable. # Notes: Telegram freeform input now includes LLM-first routing context to improve intent understanding before entity lookups. # Notes: Command payload supports async_only for automation-first queueing when immediate inline handling is not required. # Notes: Blog: https://www.vcloudinfo.com/2026/03/joanna-dispatch-telemetry-home-assistant-infrastructure-dashboard/ @@ -49,6 +50,18 @@ rest_command: "wake": {{ wake | default(false) | tojson }}, "source": "home_assistant" } + + bearclaw_duplicati_verify: + url: !secret bearclaw_duplicati_verify_url + method: post + timeout: 60 + content_type: application/json + headers: + x-codex-token: !secret bearclaw_token + payload: > + { + "reason": {{ reason | default('home_assistant') | tojson }} + } sensor: - platform: rest name: BearClaw Status Telemetry diff --git a/config/packages/infrastructure_observability.yaml b/config/packages/infrastructure_observability.yaml index a85605c6..649ce3eb 100644 --- a/config/packages/infrastructure_observability.yaml +++ b/config/packages/infrastructure_observability.yaml @@ -4,11 +4,12 @@ # Original Repo : https://github.com/CCOSTAN/Home-AssistantConfig # ------------------------------------------------------------------- # Infrastructure Observability - Normalized infra monitoring signals -# WAN/DNS/backup/website/domain/cert state normalized for dashboards. +# WAN/DNS/website/domain/cert state normalized for dashboards. # ------------------------------------------------------------------- # Related Issue: 1584 # Notes: Home dashboard consumes `infra_*` entities for exceptions-only alerts. # Notes: Domain warning threshold is <30 days; critical threshold is <14 days. +# Notes: Nightly Duplicati verification is performed by codex_appliance against the Duplicati API because HA backup entities are not available. ###################################################################### command_line: @@ -53,18 +54,6 @@ template: {{ fallback }} {% endif %} - - name: "Infra Backup Age Hours" - unique_id: infra_backup_age_hours - unit_of_measurement: "h" - state: >- - {% set stamp = states('sensor.dockerconfigs_backup_date') %} - {% set ts = as_datetime(stamp) %} - {% if ts is not none %} - {{ ((now() - ts).total_seconds() / 3600) | round(1) }} - {% else %} - {{ none }} - {% endif %} - - name: "Infra Domain Expiry Min Days" unique_id: infra_domain_expiry_min_days unit_of_measurement: "d" @@ -165,19 +154,6 @@ template: {% set lat = lat_raw | float(0) %} {{ invalid or loss > 5 or lat > 80 }} - - name: "Infra Backup Stale Or Failed" - unique_id: infra_backup_stale_or_failed - device_class: problem - state: >- - {% set status = states('sensor.dockerconfigs_backup_status') | lower %} - {% set err = states('sensor.dockerconfigs_backup_error_message') | lower %} - {% set age = states('sensor.infra_backup_age_hours') | float(9999) %} - {% set failed = status in ['failed', 'failure', 'error', 'fatal'] or - 'fail' in status or - 'error' in status or - err not in ['unknown', 'unavailable', 'none', ''] %} - {{ failed or age > 24 }} - - name: "Infra DNS Pihole Degraded" unique_id: infra_dns_pihole_degraded device_class: problem @@ -353,3 +329,85 @@ automation: continue_on_error: true data: issue_id: infra_website_latency_degraded + + - alias: "Infrastructure - Backup Nightly Verification" + id: infra_backup_nightly_verification + description: "Use codex_appliance to verify the latest Duplicati run and dispatch Joanna only on failure." + mode: single + trigger: + - platform: time + at: "06:15:00" + action: + - variables: + trigger_context: "HA automation infra_backup_nightly_verification (Infrastructure - Backup Nightly Verification)" + duplicati_state: "{{ states('switch.duplicati_container') }}" + - action: rest_command.bearclaw_duplicati_verify + data: + reason: "ha_nightly" + response_variable: duplicati_verify + - service: script.send_to_logbook + data: + topic: "BACKUP" + message: >- + {% set payload = duplicati_verify['content'] if duplicati_verify is mapping and duplicati_verify['content'] is mapping else {} %} + {% set detail = payload['detail'] if payload is mapping and payload['detail'] is mapping else {} %} + {{ detail.get('summary', 'Nightly Duplicati verification completed.') }} + - variables: + verify_payload: "{{ duplicati_verify['content'] if duplicati_verify is mapping and duplicati_verify['content'] is mapping else {} }}" + verify_detail: "{{ verify_payload['detail'] if verify_payload is mapping and verify_payload['detail'] is mapping else {} }}" + verify_http_status: "{{ duplicati_verify['status'] | int(0) if duplicati_verify is mapping else 0 }}" + verify_healthy: "{{ verify_payload.get('ok', false) and verify_detail.get('healthy', false) }}" + verify_status: "{{ verify_detail.get('status', 'unknown') }}" + verify_summary: "{{ verify_detail.get('summary', 'Duplicati verification did not return a summary.') }}" + verify_issue: "{{ verify_detail.get('issue', verify_payload.get('error', 'duplicati_verify_failed')) }}" + verify_backup_name: "{{ verify_detail.get('backupName', 'Docker_Configs') }}" + verify_latest_result: "{{ verify_detail.get('latestResult', {}) if verify_detail is mapping else {} }}" + verify_last_success: "{{ verify_detail.get('lastSuccessfulRun', {}) if verify_detail is mapping else {} }}" + - choose: + - conditions: "{{ verify_healthy }}" + sequence: + - service: repairs.remove + continue_on_error: true + data: + issue_id: infra_duplicati_backup_failure + default: + - service: repairs.create + data: + issue_id: infra_duplicati_backup_failure + title: "Duplicati nightly backup verification failed" + description: >- + {{ verify_summary }} + Backup={{ verify_backup_name }}; + status={{ verify_status }}; + last_result={{ verify_latest_result.get('endedAt', 'n/a') }}; + last_success={{ verify_last_success.get('endedAt', 'n/a') }}. + severity: error + persistent: true + - service: script.joanna_dispatch + data: + trigger_context: "{{ trigger_context }}" + source: "home_assistant_automation.infra_backup_nightly_verification" + summary: "Nightly Duplicati backup verification failed" + entity_ids: + - "switch.duplicati_container" + diagnostics: >- + scheduled_time=06:15:00, + duplicati_container={{ duplicati_state }}, + verifier_http_status={{ verify_http_status }}, + verifier_status={{ verify_status }}, + verifier_issue={{ verify_issue }}, + backup_name={{ verify_backup_name }}, + latest_result={{ verify_latest_result.get('endedAt', 'n/a') }}, + last_success={{ verify_last_success.get('endedAt', 'n/a') }} + request: >- + Investigate the Duplicati backup job {{ verify_backup_name }}. + The codex_appliance verifier reported status {{ verify_status }} with issue {{ verify_issue }}. + Use the Duplicati API or UI directly, resolve the failure if possible, and verify a successful run before closing out. + Reply with explicit status fields: + resolved=true/false, + backup_status, + last_success_time, + root_cause, + action_taken, + verification, + next_action_required=true/false. diff --git a/config/recorder.yaml b/config/recorder.yaml index c62ae635..f258e4c9 100755 --- a/config/recorder.yaml +++ b/config/recorder.yaml @@ -92,9 +92,6 @@ exclude: - sensor.clock_time - sensor.clock_time_2 - sensor.date - - sensor.dockerconfigs_backup_date - - sensor.dockerconfigs_backup_error_message - - sensor.dockerconfigs_backup_status - sensor.external_ip - sensor.ha_uptime - sensor.large_garage_door_since diff --git a/config/travis_secrets.yaml b/config/travis_secrets.yaml index db4402ad..b2411225 100755 --- a/config/travis_secrets.yaml +++ b/config/travis_secrets.yaml @@ -87,3 +87,4 @@ wolframalpha_labor_api: https://api.wolframalpha.com/v2/result?appid=JIUY8U-4V8K wolframalpha_memorial_api: https://api.wolframalpha.com/v2/result?appid=JIUY8U-4V8KY45VT1&i=How%20many%20days%20until%20memorial wolframalpha_thanksgiving_api: https://api.wolframalpha.com/v2/result?appid=JIUY8U-4V8KY45VT1&i=How%20many%20days%20until%20thanksgiving bearclaw_status_url: http://docker17.local:8124/api/bearclaw/status +bearclaw_duplicati_verify_url: http://docker17.local:8124/api/admin/actions/duplicati-verify