|
|
|
|
@ -15,6 +15,7 @@ import json
|
|
|
|
|
import shlex
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import Optional
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
MIN_INSTANCES = 2
|
|
|
|
|
@ -23,6 +24,9 @@ MAX_SERVER_ID = 0xFFFFFFFF
|
|
|
|
|
SUPPORT_SECTIONS = ('GLOBAL', 'REPORTS', 'LOGGER', 'ALIASES', 'ALLSTAR')
|
|
|
|
|
INTERNAL_WORKER_PORT_BASE = 7000
|
|
|
|
|
INTERNAL_AGGREGATOR_PORT_BASE = 7100
|
|
|
|
|
INTERNAL_AGGREGATOR_BRIDGE_PORT = 7200
|
|
|
|
|
INTERNAL_BRIDGE_PORT = 7201
|
|
|
|
|
LORO_PORT_STRIDE = 2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class HDStackConfigurationError(ValueError):
|
|
|
|
|
@ -37,6 +41,8 @@ class HDStackLayout:
|
|
|
|
|
base_server_id: int
|
|
|
|
|
aggregator_config: Path
|
|
|
|
|
worker_configs: tuple
|
|
|
|
|
loro_configs: tuple
|
|
|
|
|
bridge_config: Optional[Path]
|
|
|
|
|
supervisor_config: Path
|
|
|
|
|
backend_ranges: tuple
|
|
|
|
|
worker_report_ports: tuple
|
|
|
|
|
@ -50,23 +56,47 @@ def generate_hdstack(
|
|
|
|
|
instances,
|
|
|
|
|
base_server_id,
|
|
|
|
|
bridge_enabled=True,
|
|
|
|
|
loro_config=None,
|
|
|
|
|
):
|
|
|
|
|
"""Generate configs for an aggregator, workers, optional bridge, proxy and MUX."""
|
|
|
|
|
instances = _validated_instances(instances)
|
|
|
|
|
base_server_id = _validated_base_id(base_server_id, instances)
|
|
|
|
|
bridge_enabled = _validated_bridge_enabled(bridge_enabled)
|
|
|
|
|
main_path = _required_file(main_config, 'main FreeDMR configuration')
|
|
|
|
|
source = _read_config(main_path)
|
|
|
|
|
echo_enabled = _echo_enabled(source)
|
|
|
|
|
|
|
|
|
|
loro_path = None
|
|
|
|
|
loro_source = None
|
|
|
|
|
if echo_enabled:
|
|
|
|
|
if loro_config is None:
|
|
|
|
|
loro_config = Path(__file__).resolve().with_name('loro.cfg')
|
|
|
|
|
loro_path = _required_file(loro_config, 'Loro configuration')
|
|
|
|
|
loro_source = _read_config(loro_path)
|
|
|
|
|
_validate_loro_config(source, loro_source)
|
|
|
|
|
|
|
|
|
|
bridge_path = None
|
|
|
|
|
rules_path = None
|
|
|
|
|
bridge_source = None
|
|
|
|
|
bridge_server_id = None
|
|
|
|
|
if bridge_enabled:
|
|
|
|
|
bridge_path = _required_file(bridge_config, 'bridge configuration')
|
|
|
|
|
rules_path = _required_file(bridge_rules, 'bridge rules')
|
|
|
|
|
bridge_source = _read_config(bridge_path)
|
|
|
|
|
bridge_server_id = _section_integer(
|
|
|
|
|
bridge_source,
|
|
|
|
|
'GLOBAL',
|
|
|
|
|
'SERVER_ID',
|
|
|
|
|
0,
|
|
|
|
|
)
|
|
|
|
|
_validate_bridge_server_id(
|
|
|
|
|
bridge_server_id,
|
|
|
|
|
base_server_id,
|
|
|
|
|
instances,
|
|
|
|
|
)
|
|
|
|
|
output_path = Path(output_dir)
|
|
|
|
|
|
|
|
|
|
source = _read_config(main_path)
|
|
|
|
|
system = _system_template(source)
|
|
|
|
|
|
|
|
|
|
hbp_port = _integer(system, 'PORT', 54000, 'SYSTEM PORT')
|
|
|
|
|
generator_size = _integer(system, 'GENERATOR', 100, 'SYSTEM GENERATOR')
|
|
|
|
|
if generator_size < 1:
|
|
|
|
|
@ -83,6 +113,25 @@ def generate_hdstack(
|
|
|
|
|
_validate_port(start, 'generated HBP range start')
|
|
|
|
|
_validate_port(end, 'generated HBP range end')
|
|
|
|
|
|
|
|
|
|
loro_port_pairs = ()
|
|
|
|
|
if echo_enabled:
|
|
|
|
|
echo_peer_port = _section_integer(source, 'ECHO', 'PORT', 54916)
|
|
|
|
|
echo_master_port = _section_integer(source, 'ECHO', 'MASTER_PORT', 54915)
|
|
|
|
|
loro_port_pairs = tuple(
|
|
|
|
|
(
|
|
|
|
|
echo_peer_port + (number - 1) * LORO_PORT_STRIDE,
|
|
|
|
|
echo_master_port + (number - 1) * LORO_PORT_STRIDE,
|
|
|
|
|
)
|
|
|
|
|
for number in range(1, instances + 1)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
loro_report_ports = ()
|
|
|
|
|
if echo_enabled:
|
|
|
|
|
loro_report_port = _section_integer(loro_source, 'REPORTS', 'REPORT_PORT', 4821)
|
|
|
|
|
loro_report_ports = tuple(
|
|
|
|
|
loro_report_port + number - 1 for number in range(1, instances + 1)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
report_enabled = _boolean(source, 'REPORTS', 'REPORT', True)
|
|
|
|
|
report_port = _section_integer(source, 'REPORTS', 'REPORT_PORT', 4321)
|
|
|
|
|
_validate_port(report_port, 'report MUX port')
|
|
|
|
|
@ -102,6 +151,8 @@ def generate_hdstack(
|
|
|
|
|
worker_report_ports,
|
|
|
|
|
instances,
|
|
|
|
|
bridge_enabled,
|
|
|
|
|
loro_port_pairs,
|
|
|
|
|
loro_report_ports,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
aggregator = _support_config(source)
|
|
|
|
|
@ -125,7 +176,36 @@ def generate_hdstack(
|
|
|
|
|
base_server_id + number,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
generated_bridge = None
|
|
|
|
|
if bridge_enabled:
|
|
|
|
|
aggregator_name = 'OBP-HDSTACK-BRIDGE'
|
|
|
|
|
bridge_name = 'OBP-HDSTACK-AGGREGATOR'
|
|
|
|
|
for config_source, name in (
|
|
|
|
|
(source, aggregator_name),
|
|
|
|
|
(bridge_source, bridge_name),
|
|
|
|
|
):
|
|
|
|
|
if config_source.has_section(name):
|
|
|
|
|
raise HDStackConfigurationError(
|
|
|
|
|
'configuration already contains reserved section {}'.format(name)
|
|
|
|
|
)
|
|
|
|
|
_add_internal_openbridge(
|
|
|
|
|
aggregator,
|
|
|
|
|
aggregator_name,
|
|
|
|
|
INTERNAL_AGGREGATOR_BRIDGE_PORT,
|
|
|
|
|
INTERNAL_BRIDGE_PORT,
|
|
|
|
|
bridge_server_id,
|
|
|
|
|
)
|
|
|
|
|
generated_bridge = _clone_config(bridge_source)
|
|
|
|
|
_add_internal_openbridge(
|
|
|
|
|
generated_bridge,
|
|
|
|
|
bridge_name,
|
|
|
|
|
INTERNAL_BRIDGE_PORT,
|
|
|
|
|
INTERNAL_AGGREGATOR_BRIDGE_PORT,
|
|
|
|
|
base_server_id,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
workers = []
|
|
|
|
|
loros = []
|
|
|
|
|
for number, (start, _end) in enumerate(backend_ranges, 1):
|
|
|
|
|
worker = _support_config(source)
|
|
|
|
|
worker['GLOBAL']['SERVER_ID'] = str(base_server_id + number)
|
|
|
|
|
@ -141,6 +221,25 @@ def generate_hdstack(
|
|
|
|
|
_copy_section(source, worker, 'SYSTEM')
|
|
|
|
|
worker['SYSTEM']['PORT'] = str(start)
|
|
|
|
|
name = 'OBP-HDSTACK-AGGREGATOR'
|
|
|
|
|
if echo_enabled:
|
|
|
|
|
echo_peer_port, echo_master_port = loro_port_pairs[number - 1]
|
|
|
|
|
_copy_section(source, worker, 'ECHO')
|
|
|
|
|
worker['ECHO']['IP'] = '127.0.0.1'
|
|
|
|
|
worker['ECHO']['PORT'] = str(echo_peer_port)
|
|
|
|
|
worker['ECHO']['MASTER_IP'] = '127.0.0.1'
|
|
|
|
|
worker['ECHO']['MASTER_PORT'] = str(echo_master_port)
|
|
|
|
|
|
|
|
|
|
loro = _clone_config(loro_source)
|
|
|
|
|
loro['GLOBAL']['SERVER_ID'] = '9990'
|
|
|
|
|
loro['REPORTS']['REPORT'] = 'False'
|
|
|
|
|
loro['REPORTS']['REPORT_PORT'] = str(loro_report_ports[number - 1])
|
|
|
|
|
loro['LOGGER']['LOG_NAME'] = 'ECHO-{}'.format(number)
|
|
|
|
|
loro['ALIASES']['TRY_DOWNLOAD'] = 'False'
|
|
|
|
|
loro['PARROT']['IP'] = '127.0.0.1'
|
|
|
|
|
loro['PARROT']['PORT'] = str(echo_master_port)
|
|
|
|
|
loro['PARROT']['MAX_PEERS'] = '1'
|
|
|
|
|
loro['PARROT']['PASSPHRASE'] = worker['ECHO']['PASSPHRASE']
|
|
|
|
|
loros.append(loro)
|
|
|
|
|
if source.has_section(name):
|
|
|
|
|
raise HDStackConfigurationError(
|
|
|
|
|
'main configuration already contains reserved section {}'.format(name)
|
|
|
|
|
@ -162,14 +261,28 @@ def generate_hdstack(
|
|
|
|
|
)
|
|
|
|
|
supervisor_path = output_path / 'supervisord.conf'
|
|
|
|
|
|
|
|
|
|
loro_paths = tuple(
|
|
|
|
|
output_path / 'loro-{}.cfg'.format(number)
|
|
|
|
|
for number in range(1, instances + 1)
|
|
|
|
|
) if echo_enabled else ()
|
|
|
|
|
generated_bridge_path = (
|
|
|
|
|
output_path / 'bridge.cfg'
|
|
|
|
|
if bridge_enabled
|
|
|
|
|
else None
|
|
|
|
|
)
|
|
|
|
|
_write_config(aggregator, aggregator_path)
|
|
|
|
|
for worker, path in zip(workers, worker_paths):
|
|
|
|
|
_write_config(worker, path)
|
|
|
|
|
for loro, path in zip(loros, loro_paths):
|
|
|
|
|
_write_config(loro, path)
|
|
|
|
|
if generated_bridge_path is not None:
|
|
|
|
|
_write_config(generated_bridge, generated_bridge_path)
|
|
|
|
|
supervisor_path.write_text(
|
|
|
|
|
_supervisor_config(
|
|
|
|
|
aggregator_path,
|
|
|
|
|
worker_paths,
|
|
|
|
|
bridge_path,
|
|
|
|
|
loro_paths,
|
|
|
|
|
generated_bridge_path,
|
|
|
|
|
rules_path,
|
|
|
|
|
backend_ranges,
|
|
|
|
|
report_enabled,
|
|
|
|
|
@ -186,6 +299,8 @@ def generate_hdstack(
|
|
|
|
|
base_server_id=base_server_id,
|
|
|
|
|
aggregator_config=aggregator_path,
|
|
|
|
|
worker_configs=worker_paths,
|
|
|
|
|
loro_configs=loro_paths,
|
|
|
|
|
bridge_config=generated_bridge_path,
|
|
|
|
|
supervisor_config=supervisor_path,
|
|
|
|
|
backend_ranges=backend_ranges,
|
|
|
|
|
worker_report_ports=worker_report_ports,
|
|
|
|
|
@ -215,6 +330,38 @@ def _validated_base_id(value, instances):
|
|
|
|
|
return value
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _validate_bridge_server_id(value, base_server_id, instances):
|
|
|
|
|
if value < 1 or value > MAX_SERVER_ID:
|
|
|
|
|
raise HDStackConfigurationError(
|
|
|
|
|
'bridge GLOBAL SERVER_ID must fit an unsigned 32-bit ID'
|
|
|
|
|
)
|
|
|
|
|
if base_server_id <= value <= base_server_id + instances:
|
|
|
|
|
raise HDStackConfigurationError(
|
|
|
|
|
'bridge GLOBAL SERVER_ID conflicts with an HDStack server ID'
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _echo_enabled(source):
|
|
|
|
|
if not source.has_section('ECHO'):
|
|
|
|
|
return False
|
|
|
|
|
if not _boolean(source, 'ECHO', 'ENABLED', True):
|
|
|
|
|
return False
|
|
|
|
|
if _mode(source, 'ECHO') != 'PEER':
|
|
|
|
|
raise HDStackConfigurationError('enabled [ECHO] must use MODE PEER')
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _validate_loro_config(source, loro_source):
|
|
|
|
|
if not loro_source.has_section('PARROT'):
|
|
|
|
|
raise HDStackConfigurationError('Loro configuration requires [PARROT]')
|
|
|
|
|
if _mode(loro_source, 'PARROT') != 'MASTER':
|
|
|
|
|
raise HDStackConfigurationError('Loro [PARROT] must use MODE MASTER')
|
|
|
|
|
if not _boolean(loro_source, 'PARROT', 'ENABLED', True):
|
|
|
|
|
raise HDStackConfigurationError('Loro [PARROT] must be enabled')
|
|
|
|
|
_section_integer(source, 'ECHO', 'PORT', 54916)
|
|
|
|
|
_section_integer(source, 'ECHO', 'MASTER_PORT', 54915)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _validated_bridge_enabled(value):
|
|
|
|
|
if not isinstance(value, bool):
|
|
|
|
|
raise HDStackConfigurationError('HDSTACK_BRIDGE must be 0 or 1')
|
|
|
|
|
@ -269,6 +416,13 @@ def _copy_section(source, destination, section):
|
|
|
|
|
destination[section][key] = value
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _clone_config(source):
|
|
|
|
|
result = configparser.ConfigParser(interpolation=None)
|
|
|
|
|
for section in source.sections():
|
|
|
|
|
_copy_section(source, result, section)
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _mode(source, section):
|
|
|
|
|
return source.get(section, 'MODE', fallback='').strip().upper()
|
|
|
|
|
|
|
|
|
|
@ -331,6 +485,8 @@ def _validate_generated_port_collisions(
|
|
|
|
|
worker_report_ports,
|
|
|
|
|
instances,
|
|
|
|
|
bridge_enabled,
|
|
|
|
|
loro_port_pairs,
|
|
|
|
|
loro_report_ports,
|
|
|
|
|
):
|
|
|
|
|
generated = {}
|
|
|
|
|
|
|
|
|
|
@ -350,11 +506,19 @@ def _validate_generated_port_collisions(
|
|
|
|
|
for port in range(start, end + 1):
|
|
|
|
|
reserve(port, 'worker {} HBP'.format(number))
|
|
|
|
|
for number in range(1, instances + 1):
|
|
|
|
|
reserve(INTERNAL_WORKER_PORT_BASE + number, 'worker {} OBP'.format(number))
|
|
|
|
|
reserve(INTERNAL_WORKER_PORT_BASE + number, 'worker {} FBP'.format(number))
|
|
|
|
|
reserve(
|
|
|
|
|
INTERNAL_AGGREGATOR_PORT_BASE + number,
|
|
|
|
|
'aggregator worker {} OBP'.format(number),
|
|
|
|
|
'aggregator worker {} FBP'.format(number),
|
|
|
|
|
)
|
|
|
|
|
if bridge_enabled:
|
|
|
|
|
reserve(INTERNAL_AGGREGATOR_BRIDGE_PORT, 'aggregator bridge FBP')
|
|
|
|
|
reserve(INTERNAL_BRIDGE_PORT, 'bridge aggregator FBP')
|
|
|
|
|
for number, (peer_port, master_port) in enumerate(loro_port_pairs, 1):
|
|
|
|
|
reserve(peer_port, 'worker {} ECHO peer'.format(number))
|
|
|
|
|
reserve(master_port, 'worker {} Loro master'.format(number))
|
|
|
|
|
for number, port in enumerate(loro_report_ports, 1):
|
|
|
|
|
reserve(port, 'worker {} Loro reporting'.format(number))
|
|
|
|
|
if report_enabled:
|
|
|
|
|
reserve(report_port, 'report MUX')
|
|
|
|
|
for number, port in enumerate(worker_report_ports, 1):
|
|
|
|
|
@ -421,8 +585,8 @@ def _add_internal_openbridge(
|
|
|
|
|
'SUB_ACL': 'PERMIT:ALL',
|
|
|
|
|
'TGID_ACL': 'PERMIT:ALL',
|
|
|
|
|
'RELAX_CHECKS': 'False',
|
|
|
|
|
'ENHANCED_OBP': 'False',
|
|
|
|
|
'PROTO_VER': '1',
|
|
|
|
|
'ENHANCED_OBP': 'True',
|
|
|
|
|
'PROTO_VER': '5',
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -473,6 +637,7 @@ def _program(name, command, priority, environment=None):
|
|
|
|
|
def _supervisor_config(
|
|
|
|
|
aggregator_path,
|
|
|
|
|
worker_paths,
|
|
|
|
|
loro_paths,
|
|
|
|
|
bridge_path,
|
|
|
|
|
rules_path,
|
|
|
|
|
backend_ranges,
|
|
|
|
|
@ -506,6 +671,16 @@ def _supervisor_config(
|
|
|
|
|
20,
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
for number, path in enumerate(loro_paths, 1):
|
|
|
|
|
parts.append(
|
|
|
|
|
_program(
|
|
|
|
|
'playback-{}'.format(number),
|
|
|
|
|
'/opt/freedmr/playback.py -c {}'.format(
|
|
|
|
|
shlex.quote(str(path))
|
|
|
|
|
),
|
|
|
|
|
30,
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
if bridge_enabled:
|
|
|
|
|
parts.append(
|
|
|
|
|
_program(
|
|
|
|
|
@ -518,11 +693,6 @@ def _supervisor_config(
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
parts.extend([
|
|
|
|
|
_program(
|
|
|
|
|
'playback',
|
|
|
|
|
'/opt/freedmr/playback.py -c /opt/freedmr/loro.cfg',
|
|
|
|
|
30,
|
|
|
|
|
),
|
|
|
|
|
_program(
|
|
|
|
|
'proxy',
|
|
|
|
|
'python /opt/freedmr/hotspot_proxy_v2.py',
|
|
|
|
|
@ -563,6 +733,7 @@ def main(argv=None):
|
|
|
|
|
parser.add_argument('--bridge-config', required=True)
|
|
|
|
|
parser.add_argument('--bridge-rules', required=True)
|
|
|
|
|
parser.add_argument('--output-dir', required=True)
|
|
|
|
|
parser.add_argument('--loro-config', default='/opt/freedmr/loro.cfg')
|
|
|
|
|
parser.add_argument('--instances', required=True, type=int)
|
|
|
|
|
parser.add_argument('--base-id', required=True, type=int)
|
|
|
|
|
parser.add_argument('--bridge-enabled', choices=('0', '1'), default='1')
|
|
|
|
|
@ -576,6 +747,7 @@ def main(argv=None):
|
|
|
|
|
args.instances,
|
|
|
|
|
args.base_id,
|
|
|
|
|
bridge_enabled=args.bridge_enabled == '1',
|
|
|
|
|
loro_config=args.loro_config,
|
|
|
|
|
)
|
|
|
|
|
except HDStackConfigurationError as err:
|
|
|
|
|
parser.error(str(err))
|
|
|
|
|
|
Simon
3 weeks ago