diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ef8f26..a8c427d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ build-extrastats: # This job runs in the build stage, which runs first. - echo "Compile complete." only: - - extrastats + - extrastats2 build-testing: # This job runs in the build stage, which runs first. stage: build diff --git a/bridge_master.py b/bridge_master.py index 976a293..ead1ef4 100644 --- a/bridge_master.py +++ b/bridge_master.py @@ -331,7 +331,7 @@ def rule_timer_loop(): _bridge_used = True logger.info('(ROUTER) Conference Bridge ACTIVE (ON timer running): System: %s Bridge: %s, TS: %s, TGID: %s, Timeout in: %.2fs,', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID']), timeout_in) elif _system['ACTIVE'] == False: - logger.trace('(ROUTER) Conference Bridge INACTIVE (no change): System: %s Bridge: %s, TS: %s, TGID: %s', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID'])) + logger.debug('(ROUTER) Conference Bridge INACTIVE (no change): System: %s Bridge: %s, TS: %s, TGID: %s', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID'])) elif _system['TO_TYPE'] == 'OFF': if _system['ACTIVE'] == False: if _system['TIMER'] < _now: @@ -344,13 +344,13 @@ def rule_timer_loop(): logger.info('(ROUTER) Conference Bridge INACTIVE (OFF timer running): System: %s Bridge: %s, TS: %s, TGID: %s, Timeout in: %.2fs,', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID']), timeout_in) elif _system['ACTIVE'] == True: _bridge_used = True - logger.trace('(ROUTER) Conference Bridge ACTIVE (no change): System: %s Bridge: %s, TS: %s, TGID: %s', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID'])) + logger.debug('(ROUTER) Conference Bridge ACTIVE (no change): System: %s Bridge: %s, TS: %s, TGID: %s', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID'])) else: if _system['SYSTEM'][0:3] != 'OBP': _bridge_used = True elif _system['SYSTEM'][0:3] == 'OBP' and _system['TO_TYPE'] == 'STAT': _bridge_used = True - logger.trace('(ROUTER) Conference Bridge NO ACTION: System: %s, Bridge: %s, TS: %s, TGID: %s', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID'])) + logger.debug('(ROUTER) Conference Bridge NO ACTION: System: %s, Bridge: %s, TS: %s, TGID: %s', _system['SYSTEM'], _bridge, _system['TS'], int_id(_system['TGID'])) if _bridge_used == False: _remove_bridges.append(_bridge) @@ -923,8 +923,7 @@ def options_config(): if _options['TS1_STATIC']: ts1 = _options['TS1_STATIC'].split(',') for tg in ts1: - if not tg or int(tg) == 0 or int(tg) >= 16777215 or tg == _options['DEFAULT_REFLECTOR']: - logger.debug('(OPTIONS) %s not setting TS1 Static %s. Bad TG or conflict with DIAL',_system,tg) + if not tg: continue tg = int(tg) make_static_tg(tg,1,_tmout,_system) @@ -935,8 +934,7 @@ def options_config(): if CONFIG['SYSTEMS'][_system]['TS2_STATIC']: ts2 = CONFIG['SYSTEMS'][_system]['TS2_STATIC'].split(',') for tg in ts2: - if not tg or int(tg) == 0 or int(tg) >= 16777215 or tg == _options['DEFAULT_REFLECTOR'] or (tg and ts1 and tg in ts1): - logger.debug('(OPTIONS) %s not setting TS2 Static %s. Bad TG or conflict with DIAL or TS1',_system,tg) + if not tg or int(tg) == 0 or int(tg) >= 16777215: continue tg = int(tg) reset_static_tg(tg,2,_tmout,_system) @@ -2742,10 +2740,10 @@ if __name__ == '__main__': options = options_task.start(26) options.addErrback(loopingErrHandle) - #STAT trimmer - once every hour (roughly - shifted so all timed tasks don't run at once + #STAT trimmer - once every 10 mins (roughly - shifted so all timed tasks don't run at once if CONFIG['GLOBAL']['GEN_STAT_BRIDGES']: stat_trimmer_task = task.LoopingCall(statTrimmer) - stat_trimmer = stat_trimmer_task.start(3700)#3600 + stat_trimmer = stat_trimmer_task.start(523)#3600 stat_trimmer.addErrback(loopingErrHandle) #KA Reporting diff --git a/docker-configs/docker-compose.yml b/docker-configs/docker-compose.yml index c4c3738..a16282f 100644 --- a/docker-configs/docker-compose.yml +++ b/docker-configs/docker-compose.yml @@ -33,7 +33,7 @@ services: - '62031:62031/udp' #Change the below to inlude ports used for your OBP(s) - '62041:62041/udp' - image: 'hacknix/freedmr:latest' + image: 'gitlab.hacknix.net:5050/hacknix/freedmr:latest' restart: "unless-stopped" networks: app_net: @@ -52,45 +52,56 @@ services: #- FDPROXY_LISTENPORT=62031 read_only: "true" - freedmrmon: - container_name: freedmrmon + freedmrmonitor2: + container_name: freedmrmonitor2 cpu_shares: 512 depends_on: - freedmr - volumes: - #This should be kept to a manageable size from - #cron or logrotate outisde of the container. - - '/var/log/FreeDMRmonitor/:/opt/FreeDMRmonitor/log/' - #Write JSON files outside of container - - '/etc/freedmr/json/:/opt/FreeDMRmonitor/json/' - - #Override config file - # - '/etc/freedmr/config.py:/opt/FreeDMRmonitor/config.py' - ports: - - '9000:9000/tcp' - image: 'hacknix/freedmrmonitor:latest' + image: 'gitlab.hacknix.net:5050/freedmr/freedmrmonitor2/freedmrmonitor2:monitor-latest' restart: "unless-stopped" networks: app_net: ipv4_address: 172.16.238.20 - + read_only: "true" + logging: + driver: json-file + + freedmrmonpache: container_name: freedmrmonapache cpu_shares: 512 depends_on: - - freedmrmon - #Use to override html files - #And images - #volumes: - # - '/var/www/html/:/var/www/html/' - # - '/var/www/html/images/:/var/www/html/images/' + - freedmrmonitor2 + #where to store TLS certificates + #and acme.sh files + volumes: + - '/etc/freedmr/certs/:/opt/apachecerts/' + - '/etc/freedmr/acme.sh:/root/.acme.sh/' ports: - '80:80/tcp' - image: hacknix/freedmrmonitor-apache:latest + - '443:443/tcp' + image: 'gitlab.hacknix.net:5050/freedmr/freedmrmonitor2/freedmrmonitor2:apache-latest' restart: "unless-stopped" + environment: + #Set to 1 to enable TLS support + #you'll need to actually generate the certtificates too + #using these commands when the container is running: + + #docker exec -it freedmrmonapache gencert.sh + #docker-compose restart freedmrmonapache + + #This only needs to be done once - unless the files in the volumes above are deleted. + + #The container will handle renewing the certificates every 60 days. + + #Note -the gencert.sh script only works when the webserver is available on the default port 80 + #If it's on non-standard ports, you'll need to request the certificates manually. + - 'USE_TLS=1' networks: app_net: ipv4_address: 172.16.238.30 + logging: + driver: json-file networks: app_net: diff --git a/docker-configs/docker-compose_install.sh b/docker-configs/docker-compose_install.sh index f1a2bc1..70e76b6 100644 --- a/docker-configs/docker-compose_install.sh +++ b/docker-configs/docker-compose_install.sh @@ -21,19 +21,48 @@ echo FreeDMR Docker installer... echo Installing required packages... -apt-get -y install docker.io && +echo Install Docker Community Edition... +apt-get -y remove docker docker-engine docker.io && +apt-get -y update && +apt-get -y install apt-transport-https ca-certificates curl gnupg2 software-properties-common && +curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - && +ARCH=`/usr/bin/arch` +echo "System architecture is $ARCH" +if [ "$ARCH" == "x86_64" ] +then + ARCH="amd64" +fi +add-apt-repository \ + "deb [arch=$ARCH] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" && +apt-get -y update && +apt-get -y install docker-ce && + +echo Install Docker Compose... apt-get -y install docker-compose && -apt-get -y install conntrack && echo Set userland-proxy to false... -echo '{ "userland-proxy": false}' > /etc/docker/daemon.json && +cat < /etc/docker/daemon.json && +{ + "userland-proxy": false, + "experimental": true, + "log-driver": "json-file", + "log-opts": { + "max-size": "10m", + "max-file": "3" + } +} +EOF echo Restart docker... systemctl restart docker && echo Make config directory... mkdir /etc/freedmr && -chmod 755 /etc/freedmr && +mkdir -p /etc/freedmr/acme.sh && +mkdir -p /etc/freedmr/certs && +chmod -R 755 /etc/freedmr && echo make json directory... mkdir -p /etc/freedmr/json && @@ -54,6 +83,8 @@ GEN_STAT_BRIDGES: True ALLOW_NULL_PASSPHRASE: True ANNOUNCEMENT_LANGUAGES: SERVER_ID: 0 +DATA_GATEWAY: False +VALIDATE_SERVER_IDS: True [REPORTS] @@ -63,8 +94,8 @@ REPORT_PORT: 4321 REPORT_CLIENTS: * [LOGGER] -LOG_FILE: log/freedmr.log -LOG_HANDLERS: file-timed +LOG_FILE: /dev/null +LOG_HANDLERS: console-timed LOG_LEVEL: INFO LOG_NAME: FreeDMR @@ -74,21 +105,25 @@ PATH: ./json/ PEER_FILE: peer_ids.json SUBSCRIBER_FILE: subscriber_ids.json TGID_FILE: talkgroup_ids.json -PEER_URL: https://www.radioid.net/static/rptrs.json -SUBSCRIBER_URL: http://downloads.freedmr.uk/downloads/local_subscriber_ids.json -TGID_URL: TGID_URL: https://freedmr.cymru/talkgroups/talkgroup_ids_json.php +PEER_URL: http://freedmr-lh.gb7fr.org.uk/json/peer_ids.json +SUBSCRIBER_URL: http://freedmr-lh.gb7fr.org.uk/json/subscriber_ids.json +TGID_URL: http://freedmr-lh.gb7fr.org.uk/json/talkgroup_ids.json +LOCAL_SUBSCRIBER_FILE: local_subscriber_ids.json STALE_DAYS: 1 -LOCAL_SUBSCRIBER_FILE: local_subcriber_ids.json SUB_MAP_FILE: sub_map.pkl +SERVER_ID_URL: http://freedmr-lh.gb7fr.org.uk/json/server_ids.tsv +SERVER_ID_FILE: server_ids.tsv +TOPO_FILE: topography.json + -[MYSQL] -USE_MYSQL: False -USER: hblink -PASS: mypassword -DB: hblink -SERVER: 127.0.0.1 -PORT: 3306 -TABLE: repeaters +#Control server shared allstar instance via dial / AMI +[ALLSTAR] +ENABLED: false +USER:admin +PASS: password +SERVER: asl.example.com +PORT: 5038 +NODE: 11111 [OBP-TEST] MODE: OPENBRIDGE @@ -132,6 +167,7 @@ ANNOUNCEMENT_LANGUAGE: en_GB GENERATOR: 100 ALLOW_UNREG_ID: False PROXY_CONTROL: True +OVERRIDE_IDENT_TG: [ECHO] MODE: PEER @@ -173,31 +209,27 @@ echo "BRIDGES = {'9990': [{'SYSTEM': 'ECHO', 'TS': 2, 'TGID': 9990, 'ACTIVE': Tr echo Set perms on config directory... chown -R 54000 /etc/freedmr && -echo Setup logging... -mkdir -p /var/log/freedmr && -touch /var/log/freedmr/freedmr.log && -chown -R 54000 /var/log/freedmr && -mkdir -p /var/log/FreeDMRmonitor && -touch /var/log/FreeDMRmonitor/lastheard.log && -touch /var/log/FreeDMRmonitor/hbmon.log && -chown -R 54001 /var/log/FreeDMRmonitor && - echo Get docker-compose.yml... cd /etc/freedmr && curl https://gitlab.hacknix.net/hacknix/FreeDMR/-/raw/master/docker-configs/docker-compose.yml -o docker-compose.yml && -echo Install crontab... -cat << EOF > /etc/cron.daily/lastheard -#!/bin/bash -mv /var/log/FreeDMRmonitor/lastheard.log /var/log/FreeDMRmonitor/lastheard.log.save -/usr/bin/tail -150 /var/log/FreeDMRmonitor/lastheard.log.save > /var/log/FreeDMRmonitor/lastheard.log -mv /var/log/FreeDMRmonitor/lastheard.log /var/log/FreeDMRmonitor/lastheard.log.save -/usr/bin/tail -150 /var/log/FreeDMRmonitor/lastheard.log.save > /var/log/FreeDMRmonitor/lastheard.log -EOF + chmod 755 /etc/cron.daily/lastheard +echo Tune network stack... +cat << EOF > /etc/sysctl.conf && +net.core.rmem_default=134217728 +net.core.rmem_max=134217728 +net.core.wmem_max=134217728 +net.core.rmem_default=134217728 +net.core.netdev_max_backlog=250000 +net.netfilter.nf_conntrack_udp_timeout=15 +net.netfilter.nf_conntrack_udp_timeout_stream=35 +EOF + +/usr/sbin/sysctl -p && echo Run FreeDMR container... docker-compose up -d - +echo Read notes in /etc/freedmr/docker-compose.yml to understand how to implement extra functionality. echo FreeDMR setup complete! diff --git a/hblink.py b/hblink.py index 2c0dd21..f75c0fb 100755 --- a/hblink.py +++ b/hblink.py @@ -287,7 +287,9 @@ class OPENBRIDGE(DatagramProtocol): if compare_digest(_hash, _ckhs) and (_sockaddr == self._config['TARGET_SOCK'] or self._config['RELAX_CHECKS']): _peer_id = _data[11:15] if self._config['NETWORK_ID'] != _peer_id: - logger.error('(%s) OpenBridge packet discarded because NETWORK_ID: %s Does not match sent Peer ID: %s', self._system, int_id(self._config['NETWORK_ID']), int_id(_peer_id)) + if _stream_id not in self._laststrid: + logger.error('(%s) OpenBridge packet discarded because NETWORK_ID: %s Does not match sent Peer ID: %s', self._system, int_id(self._config['NETWORK_ID']), int_id(_peer_id)) + self._laststrid.append(_stream_id) return #This is a v1 packet, so all the extended stuff we can set to default @@ -412,11 +414,15 @@ class OPENBRIDGE(DatagramProtocol): _h.update(_packet[:69]) _ckhs = _h.digest() + + _stream_id = _data[16:20] if compare_digest(_hash, _ckhs) and (_sockaddr == self._config['TARGET_SOCK'] or self._config['RELAX_CHECKS']): _peer_id = _data[11:15] if self._config['NETWORK_ID'] != _peer_id: - logger.error('(%s) OpenBridge packet discarded because NETWORK_ID: %s Does not match sent Peer ID: %s', self._system, int_id(self._config['NETWORK_ID']), int_id(_peer_id)) + if _stream_id not in self._laststrid: + logger.error('(%s) OpenBridge packet discarded because NETWORK_ID: %s Does not match sent Peer ID: %s', self._system, int_id(self._config['NETWORK_ID']), int_id(_peer_id)) + self._laststrid.append(_stream_id) return _seq = _data[4] _rf_src = _data[5:8] @@ -433,7 +439,6 @@ class OPENBRIDGE(DatagramProtocol): _call_type = 'group' _frame_type = (_bits & 0x30) >> 4 _dtype_vseq = (_bits & 0xF) # data, 1=voice header, 2=voice terminator; voice, 0=burst A ... 5=burst F - _stream_id = _data[16:20] #logger.debug('(%s) DMRD - Seqence: %s, RF Source: %s, Destination ID: %s', self._system, int_id(_seq), int_id(_rf_src), int_id(_dst_id)) #Don't do anything if we are STUNned @@ -548,7 +553,9 @@ class OPENBRIDGE(DatagramProtocol): if compare_digest(_hash, _ckhs) and (_sockaddr == self._config['TARGET_SOCK'] or self._config['RELAX_CHECKS']): _peer_id = _data[11:15] if self._config['NETWORK_ID'] != _peer_id: - logger.error('(%s) OpenBridge packet discarded because NETWORK_ID: %s Does not match sent Peer ID: %s', self._system, int_id(self._config['NETWORK_ID']), int_id(_peer_id)) + if _stream_id not in self._laststrid: + logger.error('(%s) OpenBridge packet discarded because NETWORK_ID: %s Does not match sent Peer ID: %s', self._system, int_id(self._config['NETWORK_ID']), int_id(_peer_id)) + self._laststrid.append(_stream_id) return _seq = _data[4] _rf_src = _data[5:8] @@ -761,7 +768,7 @@ class HBSYSTEM(DatagramProtocol): # Aliased in __init__ to maintenance_loop if system is a master def master_maintenance_loop(self): - logger.debug('(%s) Master maintenance loop started', self._system) + logger.trace('(%s) Master maintenance loop started', self._system) remove_list = deque() for peer in self._peers: _this_peer = self._peers[peer] diff --git a/hotspot_proxy_v2.py b/hotspot_proxy_v2.py index d1e1efb..35bb2ee 100644 --- a/hotspot_proxy_v2.py +++ b/hotspot_proxy_v2.py @@ -312,7 +312,7 @@ if __name__ == '__main__': if 'FDPROXY_CLIENTINFO' in os.environ: ClientInfo = bool(os.environ['FDPROXY_CLIENTINFO']) if 'FDPROXY_LISTENPORT' in os.environ: - ListenPort = os.environ['FDPROXY_LISTENPORT'] + ListenPort = int(os.environ['FDPROXY_LISTENPORT']) for port in range(DestportStart,DestPortEnd+1,1): CONNTRACK[port] = False