From c6f03e6123c9246b661061dc2c1940662cef0c38 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 30 Sep 2026 10:12:32 +0000 Subject: [PATCH] Make container installer upgrade-safe --- docker-configs/check-deployment-version.sh | 7 +- docker-configs/docker-compose.yml | 2 + docker-configs/docker-compose_install.sh | 85 ++++++++---- .../upgrade-daprs-to-data-gateway.sh | 131 +++++++++++++++++- docs/hdstack.md | 48 ++++--- tests/test_data_gateway_upgrade.py | 110 +++++++++++++-- tests/test_hdstack.py | 7 +- 7 files changed, 323 insertions(+), 67 deletions(-) diff --git a/docker-configs/check-deployment-version.sh b/docker-configs/check-deployment-version.sh index ed1c8cc..0b104e0 100644 --- a/docker-configs/check-deployment-version.sh +++ b/docker-configs/check-deployment-version.sh @@ -1,7 +1,7 @@ #!/bin/sh CONFIG_FILE="${1:-/opt/freedmr/freedmr.cfg}" -REQUIRED_VERSION=1 +REQUIRED_VERSION=2 UPGRADE_SCRIPT_URL='https://gitlab.hacknix.net/hacknix/FreeDMR/-/raw/master/docker-configs/upgrade-daprs-to-data-gateway.sh' installed_version=$(awk ' @@ -54,9 +54,8 @@ Required deployment version: $REQUIRED_VERSION Pulling a new container image does not update the mounted FreeDMR configuration. Download and run the deployment upgrade script before starting this image: curl -fsSL $UPGRADE_SCRIPT_URL -o /tmp/upgrade-daprs-to-data-gateway.sh - chmod 755 /tmp/upgrade-daprs-to-data-gateway.sh - sudo /tmp/upgrade-daprs-to-data-gateway.sh --dry-run - sudo /tmp/upgrade-daprs-to-data-gateway.sh + sudo /bin/bash /tmp/upgrade-daprs-to-data-gateway.sh --dry-run + sudo /bin/bash /tmp/upgrade-daprs-to-data-gateway.sh For a deliberate custom deployment only, set FREEDMR_ALLOW_UNMIGRATED_START=1 to start without applying the upgrade. diff --git a/docker-configs/docker-compose.yml b/docker-configs/docker-compose.yml index 8914b7c..1cda66f 100644 --- a/docker-configs/docker-compose.yml +++ b/docker-configs/docker-compose.yml @@ -83,6 +83,7 @@ services: environment: FREEDMR_GATEWAY_INGRESS_MODE: fbp FREEDMR_GATEWAY_FBP_RELATIONSHIPS: '__FREEDMR_GATEWAY_FBP_RELATIONSHIPS__' + FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE: /config/data-gateway-relationships.json FREEDMR_GATEWAY_RADIOID_PATH: /data/subscriber_ids.json FREEDMR_GATEWAY_APRS_HOST: rotate.aprs2.net FREEDMR_GATEWAY_APRS_PORT: '14580' @@ -91,6 +92,7 @@ services: FREEDMR_GATEWAY_LOG_LEVEL: INFO volumes: - '/etc/freedmr/json/:/data/:ro' + - '/etc/freedmr/data-gateway-relationships.json:/config/data-gateway-relationships.json:ro' networks: app_net: ipv4_address: 172.16.238.40 diff --git a/docker-configs/docker-compose_install.sh b/docker-configs/docker-compose_install.sh index 50bb147..d604d9e 100644 --- a/docker-configs/docker-compose_install.sh +++ b/docker-configs/docker-compose_install.sh @@ -18,9 +18,35 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ############################################################################### -set -e +set -eu -BASE_URL='https://gitlab.hacknix.net/hacknix/FreeDMR/-/raw/master' +BASE_URL="${FREEDMR_INSTALL_BASE_URL:-https://gitlab.hacknix.net/hacknix/FreeDMR/-/raw/master}" +CONFIG_DIR="${FREEDMR_INSTALL_CONFIG_DIR:-/etc/freedmr}" + +existing_config=0 +existing_compose=0 +[ ! -e "$CONFIG_DIR/freedmr.cfg" ] || existing_config=1 +[ ! -e "$CONFIG_DIR/docker-compose.yml" ] || existing_compose=1 +if [ "$existing_config" -eq 1 ] || [ "$existing_compose" -eq 1 ] +then + if [ "$existing_config" -ne 1 ] || [ "$existing_compose" -ne 1 ] + then + echo "Existing FreeDMR deployment is incomplete; both freedmr.cfg and docker-compose.yml are required." >&2 + exit 2 + fi + upgrade_script=$(mktemp) + trap 'rm -f "$upgrade_script"' EXIT HUP INT TERM + curl -fsSL "$BASE_URL/docker-configs/upgrade-daprs-to-data-gateway.sh" \ + -o "$upgrade_script" + /bin/bash "$upgrade_script" --config-dir "$CONFIG_DIR" "$@" + echo 'Existing FreeDMR deployment upgrade completed; fresh-install steps were not run.' + exit 0 +fi +if [ "$#" -ne 0 ] +then + echo 'Installer arguments are accepted only when upgrading an existing deployment.' >&2 + exit 2 +fi prompt_value() { @@ -261,42 +287,45 @@ EOF systemctl restart docker echo 'Installing FreeDMR configuration...' -mkdir -p /etc/freedmr/acme.sh /etc/freedmr/certs /etc/freedmr/json -chmod -R 755 /etc/freedmr -chown 54000:54000 /etc/freedmr/json +mkdir -p "$CONFIG_DIR/acme.sh" "$CONFIG_DIR/certs" "$CONFIG_DIR/json" +chmod -R 755 "$CONFIG_DIR" +chown 54000:54000 "$CONFIG_DIR/json" -curl -fsSL "$BASE_URL/FreeDMR.cfg" -o /etc/freedmr/freedmr.cfg -sed -i "0,/^SERVER_ID:/s/^SERVER_ID:.*/SERVER_ID: $HDSTACK_BASEID/" /etc/freedmr/freedmr.cfg -sed -i '/^\[GLOBAL\]/,/^\[/ s/^DATA_GATEWAY:.*/DATA_GATEWAY: True/' /etc/freedmr/freedmr.cfg -sed -i '/^\[DATA-GATEWAY\]/,/^\[/ s/^ENABLED:.*/ENABLED: True/' /etc/freedmr/freedmr.cfg -sed -i "/^\[DATA-GATEWAY\]/,/^\[/ s|^PASSPHRASE:.*|PASSPHRASE: $FREEDMR_GATEWAY_FBP_KEY|" /etc/freedmr/freedmr.cfg -sed -i '/^\[DATA-GATEWAY\]/,/^\[/ s/^TARGET_IP:.*/TARGET_IP: freedmr-data-gateway/' /etc/freedmr/freedmr.cfg +curl -fsSL "$BASE_URL/FreeDMR.cfg" -o "$CONFIG_DIR/freedmr.cfg" +sed -i "0,/^SERVER_ID:/s/^SERVER_ID:.*/SERVER_ID: $HDSTACK_BASEID/" "$CONFIG_DIR/freedmr.cfg" +sed -i '/^\[GLOBAL\]/,/^\[/ s/^DATA_GATEWAY:.*/DATA_GATEWAY: True/' "$CONFIG_DIR/freedmr.cfg" +sed -i '/^\[DATA-GATEWAY\]/,/^\[/ s/^ENABLED:.*/ENABLED: True/' "$CONFIG_DIR/freedmr.cfg" +sed -i "/^\[DATA-GATEWAY\]/,/^\[/ s|^PASSPHRASE:.*|PASSPHRASE: $FREEDMR_GATEWAY_FBP_KEY|" "$CONFIG_DIR/freedmr.cfg" +sed -i '/^\[DATA-GATEWAY\]/,/^\[/ s/^TARGET_IP:.*/TARGET_IP: freedmr-data-gateway/' "$CONFIG_DIR/freedmr.cfg" -if [ ! -e /etc/freedmr/freedmr-bridge.cfg ] +if [ ! -e "$CONFIG_DIR/freedmr-bridge.cfg" ] then - curl -fsSL "$BASE_URL/docker-configs/freedmr-bridge.cfg" -o /etc/freedmr/freedmr-bridge.cfg + curl -fsSL "$BASE_URL/docker-configs/freedmr-bridge.cfg" -o "$CONFIG_DIR/freedmr-bridge.cfg" fi if [ "$HDSTACK_BRIDGE_VALUE" -eq 1 ] then - sed -i "0,/^SERVER_ID:/s/^SERVER_ID:.*/SERVER_ID: $HDSTACK_BRIDGE_ID/" /etc/freedmr/freedmr-bridge.cfg + sed -i "0,/^SERVER_ID:/s/^SERVER_ID:.*/SERVER_ID: $HDSTACK_BRIDGE_ID/" "$CONFIG_DIR/freedmr-bridge.cfg" fi -if [ ! -e /etc/freedmr/rules-bridge.py ] +if [ ! -e "$CONFIG_DIR/rules-bridge.py" ] then - curl -fsSL "$BASE_URL/docker-configs/rules-bridge.py" -o /etc/freedmr/rules-bridge.py + curl -fsSL "$BASE_URL/docker-configs/rules-bridge.py" -o "$CONFIG_DIR/rules-bridge.py" fi -curl -fsSL "$BASE_URL/docker-configs/docker-compose.yml" -o /etc/freedmr/docker-compose.yml -sed -i "s|^[[:space:]]*#*- HDSTACK=.*| - HDSTACK=$HDSTACK_COUNT|" /etc/freedmr/docker-compose.yml -sed -i "s|^[[:space:]]*#*- HDSTACK_BASEID=.*| - HDSTACK_BASEID=$HDSTACK_BASEID|" /etc/freedmr/docker-compose.yml -sed -i "s|^[[:space:]]*- HDSTACK_BRIDGE=.*| - HDSTACK_BRIDGE=$HDSTACK_BRIDGE_VALUE|" /etc/freedmr/docker-compose.yml -sed -i "s|__FREEDMR_GATEWAY_FBP_RELATIONSHIPS__|$FREEDMR_GATEWAY_FBP_RELATIONSHIPS|" /etc/freedmr/docker-compose.yml -sed -i "s|__FREEDMR_GATEWAY_APRS_LOGIN_CALL__|$FREEDMR_GATEWAY_APRS_LOGIN_CALL|" /etc/freedmr/docker-compose.yml -sed -i "s|__FREEDMR_GATEWAY_APRS_PASSCODE__|$FREEDMR_GATEWAY_APRS_PASSCODE|" /etc/freedmr/docker-compose.yml -docker compose -f /etc/freedmr/docker-compose.yml config --quiet -sed -i '/^# FREEDMR_DEPLOYMENT_VERSION=/d' /etc/freedmr/freedmr.cfg -sed -i '1i# FREEDMR_DEPLOYMENT_VERSION=1' /etc/freedmr/freedmr.cfg -chown -R 54000:54000 /etc/freedmr +curl -fsSL "$BASE_URL/docker-configs/docker-compose.yml" -o "$CONFIG_DIR/docker-compose.yml" +sed -i "s|^[[:space:]]*#*- HDSTACK=.*| - HDSTACK=$HDSTACK_COUNT|" "$CONFIG_DIR/docker-compose.yml" +sed -i "s|^[[:space:]]*#*- HDSTACK_BASEID=.*| - HDSTACK_BASEID=$HDSTACK_BASEID|" "$CONFIG_DIR/docker-compose.yml" +sed -i "s|^[[:space:]]*- HDSTACK_BRIDGE=.*| - HDSTACK_BRIDGE=$HDSTACK_BRIDGE_VALUE|" "$CONFIG_DIR/docker-compose.yml" +sed -i "s|__FREEDMR_GATEWAY_FBP_RELATIONSHIPS__|$FREEDMR_GATEWAY_FBP_RELATIONSHIPS|" "$CONFIG_DIR/docker-compose.yml" +sed -i "s|__FREEDMR_GATEWAY_APRS_LOGIN_CALL__|$FREEDMR_GATEWAY_APRS_LOGIN_CALL|" "$CONFIG_DIR/docker-compose.yml" +sed -i "s|__FREEDMR_GATEWAY_APRS_PASSCODE__|$FREEDMR_GATEWAY_APRS_PASSCODE|" "$CONFIG_DIR/docker-compose.yml" +printf '%s\n' "$FREEDMR_GATEWAY_FBP_RELATIONSHIPS" > \ + "$CONFIG_DIR/data-gateway-relationships.json" +chmod 644 "$CONFIG_DIR/data-gateway-relationships.json" +docker compose -f "$CONFIG_DIR/docker-compose.yml" config --quiet +sed -i '/^# FREEDMR_DEPLOYMENT_VERSION=/d' "$CONFIG_DIR/freedmr.cfg" +sed -i '1i# FREEDMR_DEPLOYMENT_VERSION=2' "$CONFIG_DIR/freedmr.cfg" +chown -R 54000:54000 "$CONFIG_DIR" if [ -e /etc/cron.daily/lastheard ] then @@ -315,7 +344,7 @@ EOF /usr/sbin/sysctl -p echo 'Starting the FreeDMR container...' -cd /etc/freedmr +cd "$CONFIG_DIR" docker compose up -d echo 'Read /etc/freedmr/docker-compose.yml and docs/hdstack.md before adding optional services or custom links.' diff --git a/docker-configs/upgrade-daprs-to-data-gateway.sh b/docker-configs/upgrade-daprs-to-data-gateway.sh index b558c4b..cd0da54 100755 --- a/docker-configs/upgrade-daprs-to-data-gateway.sh +++ b/docker-configs/upgrade-daprs-to-data-gateway.sh @@ -6,7 +6,7 @@ set -eu CONFIG_DIR=/etc/freedmr DRY_RUN=0 ASSUME_YES=0 -TARGET_DEPLOYMENT_VERSION=1 +TARGET_DEPLOYMENT_VERSION=2 LOCK_DIR= WORK_DIR= @@ -22,6 +22,9 @@ operation is idempotent and can resume an unambiguous partial migration. Existing server, HDStack and APRS values are read from freedmr.cfg and the legacy D-APRS service in docker-compose.yml. A new private FBP key is generated unless freedmr.cfg already contains a non-placeholder Data Gateway passphrase. +Native relationship arrays are preserved and written to the mounted +data-gateway-relationships.json document. The three deployment files are +validated and installed transactionally with timestamped backups. USAGE } @@ -220,6 +223,7 @@ done FREEDMR_CONFIG="$CONFIG_DIR/freedmr.cfg" COMPOSE_FILE="$CONFIG_DIR/docker-compose.yml" +RELATIONSHIPS_FILE="$CONFIG_DIR/data-gateway-relationships.json" [ -f "$FREEDMR_CONFIG" ] || fail "$FREEDMR_CONFIG was not found" [ -f "$COMPOSE_FILE" ] || fail "$COMPOSE_FILE was not found" @@ -394,16 +398,19 @@ do fi done -RELATIONSHIPS=$(build_relationships \ - "$HDSTACK_COUNT" "$BASE_ID" "$FBP_KEY" \ - "$GATEWAY_BIND_PORT" "$GATEWAY_PEER_PORT") -if [ "$native_count" -eq 1 ] && [ "$NATIVE_RELATIONSHIPS" != "$RELATIONSHIPS" ] +if [ "$native_count" -eq 1 ] then - fail "the native data-gateway relationships do not match the configured HDStack layout" + RELATIONSHIPS="$NATIVE_RELATIONSHIPS" +else + RELATIONSHIPS=$(build_relationships \ + "$HDSTACK_COUNT" "$BASE_ID" "$FBP_KEY" \ + "$GATEWAY_BIND_PORT" "$GATEWAY_PEER_PORT") fi WORK_DIR=$(mktemp -d "$CONFIG_DIR/.data-gateway-upgrade.XXXXXX") PROPOSED_CONFIG="$WORK_DIR/freedmr.cfg" PROPOSED_COMPOSE="$WORK_DIR/docker-compose.yml" +PROPOSED_RELATIONSHIPS="$WORK_DIR/data-gateway-relationships.json" +printf '%s\n' "$RELATIONSHIPS" > "$PROPOSED_RELATIONSHIPS" awk -v gateway_key="$FBP_KEY" \ -v gateway_port="$GATEWAY_PEER_PORT" \ @@ -493,6 +500,7 @@ then print " environment:" print " FREEDMR_GATEWAY_INGRESS_MODE: fbp" print " FREEDMR_GATEWAY_FBP_RELATIONSHIPS: \047" relationships "\047" + print " FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE: /config/data-gateway-relationships.json" print " FREEDMR_GATEWAY_RADIOID_PATH: /data/subscriber_ids.json" print " FREEDMR_GATEWAY_APRS_HOST: rotate.aprs2.net" print " FREEDMR_GATEWAY_APRS_PORT: \04714580\047" @@ -501,6 +509,7 @@ then print " FREEDMR_GATEWAY_LOG_LEVEL: INFO" print " volumes:" print " - \047/etc/freedmr/json/:/data/:ro\047" + print " - \047/etc/freedmr/data-gateway-relationships.json:/config/data-gateway-relationships.json:ro\047" print " networks:" print " app_net:" print " ipv4_address: 172.16.238.40" @@ -543,6 +552,76 @@ else ' "$COMPOSE_FILE" > "$PROPOSED_COMPOSE" fi +COMPOSE_WITH_RELATIONSHIPS_FILE="$WORK_DIR/docker-compose.with-relationships-file.yml" +awk ' + function flush_gateway( i, inline_index, volumes_index, networks_index, + file_seen, mount_seen, list_environment) { + if (!gateway_count) + return + for (i = 1; i <= gateway_count; i++) { + if (gateway[i] ~ /FREEDMR_GATEWAY_FBP_RELATIONSHIPS:/ || + gateway[i] ~ /FREEDMR_GATEWAY_FBP_RELATIONSHIPS=/) { + if (gateway[i] !~ /FBP_RELATIONSHIPS_FILE/) + inline_index = i + if (gateway[i] ~ /^[[:space:]]*-/) + list_environment = 1 + } + if (gateway[i] ~ /FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE/) + file_seen = 1 + if (gateway[i] ~ /^[[:space:]]*volumes:[[:space:]]*$/) + volumes_index = i + if (gateway[i] ~ /^[[:space:]]*networks:[[:space:]]*$/ && !networks_index) + networks_index = i + if (gateway[i] ~ /data-gateway-relationships\.json:/) + mount_seen = 1 + } + if (!file_seen && !inline_index) { + print "Upgrade aborted: native data-gateway service has no inline relationship setting" > "/dev/stderr" + exit 2 + } + for (i = 1; i <= gateway_count; i++) { + if (!mount_seen && !volumes_index && networks_index == i) { + print " volumes:" + print " - \047/etc/freedmr/data-gateway-relationships.json:/config/data-gateway-relationships.json:ro\047" + } + print gateway[i] + if (!file_seen && inline_index == i) { + if (list_environment) + print " - FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE=/config/data-gateway-relationships.json" + else + print " FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE: /config/data-gateway-relationships.json" + } + if (!mount_seen && volumes_index == i) + print " - \047/etc/freedmr/data-gateway-relationships.json:/config/data-gateway-relationships.json:ro\047" + } + if (!mount_seen && !volumes_index && !networks_index) { + print " volumes:" + print " - \047/etc/freedmr/data-gateway-relationships.json:/config/data-gateway-relationships.json:ro\047" + } + delete gateway + gateway_count = 0 + } + /^ data-gateway:[[:space:]]*$/ { + flush_gateway() + in_gateway = 1 + gateway[++gateway_count] = $0 + next + } + in_gateway && (/^ [A-Za-z0-9_.-]+:[[:space:]]*$/ || + /^[^[:space:]#][^:]*:[[:space:]]*$/) { + flush_gateway() + in_gateway = 0 + } + in_gateway { + gateway[++gateway_count] = $0 + next + } + { print } + END { flush_gateway() } +' "$PROPOSED_COMPOSE" > "$COMPOSE_WITH_RELATIONSHIPS_FILE" || \ + fail "could not add the Data Gateway relationship file to Compose" +mv "$COMPOSE_WITH_RELATIONSHIPS_FILE" "$PROPOSED_COMPOSE" + for required in \ 'DATA_GATEWAY: True' \ '[DATA-GATEWAY]' \ @@ -562,6 +641,12 @@ if grep -q '__FREEDMR_GATEWAY_' "$PROPOSED_COMPOSE" then fail "generated Compose file contains an unreplaced gateway placeholder" fi +grep -q 'FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE:' "$PROPOSED_COMPOSE" || \ + grep -q 'FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE=' "$PROPOSED_COMPOSE" || \ + fail "generated Compose file does not select the relationship document" +grep -q 'data-gateway-relationships.json:/config/data-gateway-relationships.json:ro' \ + "$PROPOSED_COMPOSE" || \ + fail "generated Compose file does not mount the relationship document" if ! docker compose -f "$PROPOSED_COMPOSE" config --quiet >/dev/null then @@ -569,7 +654,9 @@ then fi if cmp -s "$FREEDMR_CONFIG" "$PROPOSED_CONFIG" && \ - cmp -s "$COMPOSE_FILE" "$PROPOSED_COMPOSE" + cmp -s "$COMPOSE_FILE" "$PROPOSED_COMPOSE" && \ + [ -f "$RELATIONSHIPS_FILE" ] && \ + cmp -s "$RELATIONSHIPS_FILE" "$PROPOSED_RELATIONSHIPS" then echo "FreeDMR deployment is already at version $TARGET_DEPLOYMENT_VERSION; no files were changed." exit 0 @@ -580,6 +667,7 @@ then echo "Dry run successful; no files were changed." echo "Configuration: $FREEDMR_CONFIG" echo "Compose file: $COMPOSE_FILE" + echo "Relationships: $RELATIONSHIPS_FILE" echo "Workers: $HDSTACK_COUNT" echo "Base ID: $BASE_ID" exit 0 @@ -597,24 +685,49 @@ fi stamp="$(date -u +%Y%m%dT%H%M%SZ)-$$" CONFIG_BACKUP="$FREEDMR_CONFIG.before-data-gateway.$stamp" COMPOSE_BACKUP="$COMPOSE_FILE.before-data-gateway.$stamp" +RELATIONSHIPS_BACKUP= [ ! -e "$CONFIG_BACKUP" ] || fail "backup already exists: $CONFIG_BACKUP" [ ! -e "$COMPOSE_BACKUP" ] || fail "backup already exists: $COMPOSE_BACKUP" cp -p "$FREEDMR_CONFIG" "$CONFIG_BACKUP" cp -p "$COMPOSE_FILE" "$COMPOSE_BACKUP" +if [ -e "$RELATIONSHIPS_FILE" ] +then + RELATIONSHIPS_BACKUP="$RELATIONSHIPS_FILE.before-data-gateway.$stamp" + [ ! -e "$RELATIONSHIPS_BACKUP" ] || fail "backup already exists: $RELATIONSHIPS_BACKUP" + cp -p "$RELATIONSHIPS_FILE" "$RELATIONSHIPS_BACKUP" +fi chmod --reference="$FREEDMR_CONFIG" "$PROPOSED_CONFIG" chmod --reference="$COMPOSE_FILE" "$PROPOSED_COMPOSE" +chmod 644 "$PROPOSED_RELATIONSHIPS" if [ "$(id -u)" -eq 0 ] then chown --reference="$FREEDMR_CONFIG" "$PROPOSED_CONFIG" chown --reference="$COMPOSE_FILE" "$PROPOSED_COMPOSE" + chown --reference="$FREEDMR_CONFIG" "$PROPOSED_RELATIONSHIPS" +fi +if ! mv "$PROPOSED_RELATIONSHIPS" "$RELATIONSHIPS_FILE" +then + fail "could not install the Data Gateway relationship document" fi if ! mv "$PROPOSED_CONFIG" "$FREEDMR_CONFIG" then + if [ -n "$RELATIONSHIPS_BACKUP" ] + then + cp -p "$RELATIONSHIPS_BACKUP" "$RELATIONSHIPS_FILE" + else + rm -f "$RELATIONSHIPS_FILE" + fi fail "could not install the upgraded FreeDMR configuration" fi if ! mv "$PROPOSED_COMPOSE" "$COMPOSE_FILE" then cp -p "$CONFIG_BACKUP" "$FREEDMR_CONFIG" + if [ -n "$RELATIONSHIPS_BACKUP" ] + then + cp -p "$RELATIONSHIPS_BACKUP" "$RELATIONSHIPS_FILE" + else + rm -f "$RELATIONSHIPS_FILE" + fi fail "could not install the upgraded Compose file; the FreeDMR configuration was restored" fi @@ -622,6 +735,10 @@ echo "FreeDMR deployment upgrade to version $TARGET_DEPLOYMENT_VERSION complete. echo "Backups:" echo " $CONFIG_BACKUP" echo " $COMPOSE_BACKUP" +if [ -n "$RELATIONSHIPS_BACKUP" ] +then + echo " $RELATIONSHIPS_BACKUP" +fi echo "Review the files, then run:" echo " cd $CONFIG_DIR" echo " docker compose pull freedmr data-gateway" diff --git a/docs/hdstack.md b/docs/hdstack.md index e5735cf..39c11d4 100644 --- a/docs/hdstack.md +++ b/docs/hdstack.md @@ -146,10 +146,14 @@ existing `[ECHO]` peer and master ports, normally `54916` and `54915`. New installations include a separate FreeDMR Data Gateway container for D-APRS and future packet-data services. The installer asks for the APRS-IS login callsign and passcode, generates a private FBP key, enables the -`[DATA-GATEWAY]` relationship, and writes matching relationships into the -Compose service. Existing configurations keep `DATA_GATEWAY: False` and the -disabled section by default, so replacing only the FreeDMR image does not -remove the legacy D-APRS path. +`[DATA-GATEWAY]` relationship, and writes matching relationships to +`/etc/freedmr/data-gateway-relationships.json`. The Compose service mounts +that document read-only. During the compatibility transition the same value +also remains in the established environment setting, allowing the current +stable Data Gateway image to run while development images verify that both +forms describe the same relationships. Existing configurations keep +`DATA_GATEWAY: False` and the disabled section by default, so replacing only +the FreeDMR image does not remove the legacy D-APRS path. Each worker connects directly to the gateway. The aggregator and optional `bridge.py` process do not. The default ports are: @@ -178,6 +182,13 @@ would address the FreeDMR container itself, not a separate gateway container. ### Migrating the legacy D-APRS container +Running the maintained installer on a directory which already contains both +`freedmr.cfg` and `docker-compose.yml` delegates directly to the upgrader. It +does not reinstall Docker, rewrite the Docker daemon configuration, ask +fresh-install questions or replace either file with a distributed template. +An incomplete deployment containing only one of those files is rejected for +manual review. + `upgrade-daprs-to-data-gateway.sh` migrates the documented legacy `D-APRS` Compose service to the native Data Gateway without replacing unrelated FreeDMR configuration or Compose services. This deployment transition is not @@ -189,9 +200,8 @@ and run a dry run before pulling and starting the new images: curl -fsSL \ https://gitlab.hacknix.net/hacknix/FreeDMR/-/raw/master/docker-configs/upgrade-daprs-to-data-gateway.sh \ -o /tmp/upgrade-daprs-to-data-gateway.sh -chmod 755 /tmp/upgrade-daprs-to-data-gateway.sh -sudo /tmp/upgrade-daprs-to-data-gateway.sh --dry-run -sudo /tmp/upgrade-daprs-to-data-gateway.sh +sudo /bin/bash /tmp/upgrade-daprs-to-data-gateway.sh --dry-run +sudo /bin/bash /tmp/upgrade-daprs-to-data-gateway.sh ``` The script reads the server ID and any existing Data Gateway ports and @@ -199,18 +209,22 @@ non-placeholder passphrase from `freedmr.cfg`. It reads the HDStack settings and APRS credentials from the legacy or native service in `docker-compose.yml`. If `HDSTACK` is omitted, it applies the container's CPU-count-minus-one default, clamped to one through five. A private FBP key is -generated only when no usable one exists. Missing, conflicting or invalid -existing values cause the migration to stop rather than guess. +generated only when no usable one exists. An existing native relationship +array is preserved as deployment configuration rather than rebuilt from a +canonical HDStack assumption. Missing, conflicting or invalid identity, key +or port values still cause the migration to stop rather than guess. -Before changing either file, the script validates the proposed Compose +Before changing the deployment files, the script validates the proposed Compose configuration and asks for confirmation. It uses a deployment lock and creates -timestamped backups beside `/etc/freedmr/freedmr.cfg` and -`/etc/freedmr/docker-compose.yml`. The operation is idempotent. It recognises a -complete native deployment and can resume when either the configuration or -Compose half of this migration was installed first. A complete native service -also takes precedence over a stale legacy service left beside it. Conflicting -keys, ports, malformed services and unrecognised layouts still stop for manual -review. The deployment marker is written only as part of a validated upgrade. +timestamped backups beside `/etc/freedmr/freedmr.cfg`, +`/etc/freedmr/docker-compose.yml` and an existing relationship document. The +operation is idempotent. It recognises a complete native deployment and can +resume when either the configuration or Compose half of this migration was +installed first. A complete native service also takes precedence over a stale +legacy service left beside it. Conflicting keys, ports, malformed services and +unrecognised layouts still stop for manual review. Deployment format 2 records +the mounted relationship document and is written only as part of a validated +upgrade. The script does not pull images or restart containers; follow the commands it prints after reviewing the generated files. If a current FreeDMR image is diff --git a/tests/test_data_gateway_upgrade.py b/tests/test_data_gateway_upgrade.py index ef7224d..9805518 100644 --- a/tests/test_data_gateway_upgrade.py +++ b/tests/test_data_gateway_upgrade.py @@ -11,6 +11,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "docker-configs" / "upgrade-daprs-to-data-gateway.sh" DEPLOYMENT_CHECK = ROOT / "docker-configs" / "check-deployment-version.sh" +INSTALLER = ROOT / "docker-configs" / "docker-compose_install.sh" class DataGatewayUpgradeTests(unittest.TestCase): @@ -35,10 +36,21 @@ class DataGatewayUpgradeTests(unittest.TestCase): encoding="utf-8", ) docker.chmod(0o755) + curl = bin_dir / "curl" + curl.write_text( + "#!/bin/sh\n" + "while [ \"$#\" -gt 0 ]; do\n" + " if [ \"$1\" = -o ]; then output=$2; shift 2; else shift; fi\n" + "done\n" + "cp \"$FREEDMR_TEST_UPGRADER\" \"$output\"\n", + encoding="utf-8", + ) + curl.chmod(0o755) self.environment = os.environ.copy() self.environment["PATH"] = ( str(bin_dir) + os.pathsep + self.environment["PATH"] ) + self.environment["FREEDMR_TEST_UPGRADER"] = str(SCRIPT) def run_upgrade(self, *arguments): return subprocess.run( @@ -49,6 +61,18 @@ class DataGatewayUpgradeTests(unittest.TestCase): env=self.environment, ) + def run_installer(self, *arguments): + environment = dict(self.environment) + environment["FREEDMR_INSTALL_CONFIG_DIR"] = str(self.config_dir) + environment["FREEDMR_INSTALL_BASE_URL"] = "https://unused.invalid" + return subprocess.run( + ["/bin/bash", str(INSTALLER), *arguments], + check=False, + capture_output=True, + text=True, + env=environment, + ) + def test_migrates_using_values_from_config_and_compose(self): result = self.run_upgrade("--yes") @@ -92,6 +116,18 @@ class DataGatewayUpgradeTests(unittest.TestCase): generated_key = parser.get("DATA-GATEWAY", "PASSPHRASE") self.assertRegex(generated_key, r"^[0-9a-f]{20}$") self.assertTrue(all(item["key"] == generated_key for item in relationships)) + relationship_file = self.config_dir / "data-gateway-relationships.json" + self.assertEqual(json.loads(relationship_file.read_text()), relationships) + self.assertIn( + "FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE: " + "/config/data-gateway-relationships.json", + compose, + ) + self.assertIn( + "/etc/freedmr/data-gateway-relationships.json:" + "/config/data-gateway-relationships.json:ro", + compose, + ) self.assertEqual( len(list(self.config_dir.glob("freedmr.cfg.before-data-gateway.*"))), 1, @@ -223,7 +259,7 @@ class DataGatewayUpgradeTests(unittest.TestCase): second = self.run_upgrade("--yes") self.assertEqual(second.returncode, 0, second.stderr) - self.assertIn("already at version 1", second.stdout) + self.assertIn("already at version 2", second.stdout) self.assertEqual(self.config.read_bytes(), original_config) self.assertEqual(self.compose.read_bytes(), original_compose) self.assertEqual( @@ -247,7 +283,7 @@ class DataGatewayUpgradeTests(unittest.TestCase): second = self.run_upgrade("--yes") self.assertEqual(second.returncode, 0, second.stderr) - self.assertIn("already at version 1", second.stdout) + self.assertIn("already at version 2", second.stdout) self.assertEqual(self.config.read_bytes(), original_config) self.assertEqual(self.compose.read_bytes(), original_compose) @@ -282,7 +318,7 @@ class DataGatewayUpgradeTests(unittest.TestCase): self.assertEqual(parser.get("DATA-GATEWAY", "PASSPHRASE"), expected_key) self.assertTrue(parser.getboolean("DATA-GATEWAY", "ENABLED")) self.assertIn( - "# FREEDMR_DEPLOYMENT_VERSION=1", + "# FREEDMR_DEPLOYMENT_VERSION=2", self.config.read_text(encoding="utf-8"), ) @@ -313,7 +349,7 @@ class DataGatewayUpgradeTests(unittest.TestCase): self.assertEqual(result.returncode, 2) self.assertIn("another FreeDMR deployment upgrade", result.stderr) - def test_refuses_conflicting_native_relationship_layout(self): + def test_preserves_existing_native_relationship_layout(self): first = self.run_upgrade("--yes") self.assertEqual(first.returncode, 0, first.stderr) self.compose.write_text( @@ -323,15 +359,71 @@ class DataGatewayUpgradeTests(unittest.TestCase): ), encoding="utf-8", ) - original_config = self.config.read_bytes() - original_compose = self.compose.read_bytes() + result = self.run_upgrade("--yes") + + self.assertEqual(result.returncode, 0, result.stderr) + relationships = self._relationships(self.compose.read_text(encoding="utf-8")) + self.assertEqual(relationships[1]["peer_network_id"], 99999) + relationship_file = self.config_dir / "data-gateway-relationships.json" + self.assertEqual(json.loads(relationship_file.read_text()), relationships) + + def test_upgrades_version_one_native_complex_relationships(self): + first = self.run_upgrade("--yes") + self.assertEqual(first.returncode, 0, first.stderr) + relationship_file = self.config_dir / "data-gateway-relationships.json" + relationship_file.unlink() + self.config.write_text( + self.config.read_text(encoding="utf-8").replace( + "# FREEDMR_DEPLOYMENT_VERSION=2", + "# FREEDMR_DEPLOYMENT_VERSION=1", + ), + encoding="utf-8", + ) + compose = self.compose.read_text(encoding="utf-8") + compose = compose.replace( + " FREEDMR_GATEWAY_FBP_RELATIONSHIPS_FILE: " + "/config/data-gateway-relationships.json\n", + "", + ).replace( + " - '/etc/freedmr/data-gateway-relationships.json:" + "/config/data-gateway-relationships.json:ro'\n", + "", + ) + compose = compose.replace( + '"peer_network_id":23402', + '"peer_network_id":99999', + ) + self.compose.write_text(compose, encoding="utf-8") result = self.run_upgrade("--yes") + self.assertEqual(result.returncode, 0, result.stderr) + updated = self.compose.read_text(encoding="utf-8") + relationships = self._relationships(updated) + self.assertEqual(relationships[1]["peer_network_id"], 99999) + self.assertEqual(json.loads(relationship_file.read_text()), relationships) + self.assertIn("# FREEDMR_DEPLOYMENT_VERSION=2", self.config.read_text()) + + def test_installer_delegates_existing_deployment_to_upgrader(self): + result = self.run_installer("--yes") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("deployment upgrade to version 2 complete", result.stdout) + self.assertIn("fresh-install steps were not run", result.stdout) + self.assertNotIn("Installing required packages", result.stdout) + self.assertTrue( + (self.config_dir / "data-gateway-relationships.json").is_file() + ) + + def test_installer_refuses_incomplete_existing_deployment(self): + self.compose.unlink() + original_config = self.config.read_bytes() + + result = self.run_installer("--yes") + self.assertEqual(result.returncode, 2) - self.assertIn("do not match the configured HDStack layout", result.stderr) + self.assertIn("deployment is incomplete", result.stderr) self.assertEqual(self.config.read_bytes(), original_config) - self.assertEqual(self.compose.read_bytes(), original_compose) def test_refuses_unrecognised_legacy_service_without_changes(self): self.compose.write_text( @@ -379,7 +471,7 @@ class DeploymentVersionCheckTests(unittest.TestCase): def test_current_deployment_is_accepted_silently(self): self.config.write_text( - "# FREEDMR_DEPLOYMENT_VERSION=1\n[GLOBAL]\n", + "# FREEDMR_DEPLOYMENT_VERSION=2\n[GLOBAL]\n", encoding="utf-8", ) diff --git a/tests/test_hdstack.py b/tests/test_hdstack.py index 0cfcf6e..4dd3cae 100644 --- a/tests/test_hdstack.py +++ b/tests/test_hdstack.py @@ -533,8 +533,11 @@ class HDStackContainerTests(unittest.TestCase): self.assertIn('TARGET_IP: freedmr-data-gateway', installer) self.assertIn('s/^SERVER_ID:.*/SERVER_ID: $HDSTACK_BASEID/', installer) self.assertIn('s/^SERVER_ID:.*/SERVER_ID: $HDSTACK_BRIDGE_ID/', installer) - self.assertIn('docker compose -f /etc/freedmr/docker-compose.yml config --quiet', installer) - self.assertIn("1i# FREEDMR_DEPLOYMENT_VERSION=1", installer) + self.assertIn('docker compose -f "$CONFIG_DIR/docker-compose.yml" config --quiet', installer) + self.assertIn("1i# FREEDMR_DEPLOYMENT_VERSION=2", installer) + self.assertIn('FREEDMR_INSTALL_CONFIG_DIR', installer) + self.assertIn('upgrade-daprs-to-data-gateway.sh', installer) + self.assertIn('data-gateway-relationships.json', installer) self.assertNotIn('/etc/freedmr/.env', installer) def test_installer_uses_current_debian_docker_repository(self):