From ee5814ef28529c267cfd8e8f1e7cae262d7646c5 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 1 Jul 2022 02:15:52 +0100 Subject: [PATCH 01/10] use path for SUB_MAP_FILE --- bridge_master.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridge_master.py b/bridge_master.py index 4018a84..bd5a130 100644 --- a/bridge_master.py +++ b/bridge_master.py @@ -402,7 +402,7 @@ def kaReporting(): #Write SUB_MAP to disk def subMapWrite(): try: - _fh = open(CONFIG['ALIASES']['SUB_MAP_FILE'],'wb') + _fh = open(CONFIG['ALIASES']['PATH'] + CONFIG['ALIASES']['SUB_MAP_FILE'],'wb') pickle.dump(SUB_MAP,_fh) _fh.close() logger.info('(SUBSCRIBER) Writing SUB_MAP to disk') @@ -2954,7 +2954,7 @@ if __name__ == '__main__': if CONFIG['ALIASES']['SUB_MAP_FILE']: try: - with open(CONFIG['ALIASES']['SUB_MAP_FILE'],'rb') as _fh: + with open(CONFIG['ALIASES']['PATH'] + CONFIG['ALIASES']['SUB_MAP_FILE'],'rb') as _fh: SUB_MAP = pickle.load(_fh) except: logger.warning('(SUBSCRIBER) Cannot load SUB_MAP file') From 33d99ea6564e309e90bd1eb894706fba221883d3 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 1 Jul 2022 02:34:23 +0100 Subject: [PATCH 02/10] Tidy up docker install - use directories for JSON and logs. --- docker-configs/docker-compose.yml | 15 +++++---------- docker-configs/docker-compose_install.sh | 19 ++++++------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/docker-configs/docker-compose.yml b/docker-configs/docker-compose.yml index 528d3b6..c4c3738 100644 --- a/docker-configs/docker-compose.yml +++ b/docker-configs/docker-compose.yml @@ -24,13 +24,11 @@ services: mem_reservation: 600m volumes: - '/etc/freedmr/freedmr.cfg:/opt/freedmr/freedmr.cfg' - - '/var/log/freedmr/freedmr.log:/opt/freedmr/freedmr.log' + - '/var/log/freedmr/:/opt/freedmr/log/' - '/etc/freedmr/rules.py:/opt/freedmr/rules.py' #Write JSON files outside of container - - '/etc/freedmr/json/talkgroup_ids.json:/opt/freedmr/talkgroup_ids.json' - - '/etc/freedmr/json/subscriber_ids.json:/opt/freedmr/subscriber_ids.json' - - '/etc/freedmr/json/peer_ids.json:/opt/freedmr/peer_ids.json' - - '/etc/freedmr/json/sub_map.pkl:/opt/freedmr/sub_map.pkl' + - '/etc/freedmr/json/:/opt/freedmr/json/' + ports: - '62031:62031/udp' #Change the below to inlude ports used for your OBP(s) @@ -62,12 +60,9 @@ services: volumes: #This should be kept to a manageable size from #cron or logrotate outisde of the container. - - '/var/log/FreeDMRmonitor/lastheard.log:/opt/FreeDMRmonitor/log/lastheard.log' - - '/var/log/FreeDMRmonitor/hbmon.log:/opt/FreeDMRmonitor/log/hbmon.log' + - '/var/log/FreeDMRmonitor/:/opt/FreeDMRmonitor/log/' #Write JSON files outside of container - - '/etc/freedmr/json/talkgroup_ids.json:/opt/FreeDMRmonitor/talkgroup_ids.json' - - '/etc/freedmr/json/subscriber_ids.json:/opt/FreeDMRmonitor/subscriber_ids.json' - - '/etc/freedmr/json/peer_ids.json:/opt/FreeDMRmonitor/peer_ids.json' + - '/etc/freedmr/json/:/opt/FreeDMRmonitor/json/' #Override config file # - '/etc/freedmr/config.py:/opt/FreeDMRmonitor/config.py' diff --git a/docker-configs/docker-compose_install.sh b/docker-configs/docker-compose_install.sh index 6613396..f1a2bc1 100644 --- a/docker-configs/docker-compose_install.sh +++ b/docker-configs/docker-compose_install.sh @@ -37,14 +37,7 @@ chmod 755 /etc/freedmr && echo make json directory... mkdir -p /etc/freedmr/json && - -echo get json files... -cd /etc/freedmr/json && -curl http://downloads.freedmr.uk/downloads/local_subscriber_ids.json -o subscriber_ids.json && -curl http://downloads.freedmr.uk/downloads/talkgroup_ids.json -o talkgroup_ids.json && -curl https://www.radioid.net/static/rptrs.json -o peer_ids.json && -touch /etc/freedmr/json/sub_map.pkl && -chmod -R 777 /etc/freedmr/json && +chown 54000:54000 /etc/freedmr/json && echo Install /etc/freedmr/freedmr.cfg ... cat << EOF > /etc/freedmr/freedmr.cfg @@ -70,21 +63,21 @@ REPORT_PORT: 4321 REPORT_CLIENTS: * [LOGGER] -LOG_FILE: freedmr.log +LOG_FILE: log/freedmr.log LOG_HANDLERS: file-timed LOG_LEVEL: INFO LOG_NAME: FreeDMR [ALIASES] -TRY_DOWNLOAD: False -PATH: ./ +TRY_DOWNLOAD: True +PATH: ./json/ PEER_FILE: peer_ids.json SUBSCRIBER_FILE: subscriber_ids.json TGID_FILE: talkgroup_ids.json PEER_URL: https://www.radioid.net/static/rptrs.json SUBSCRIBER_URL: http://downloads.freedmr.uk/downloads/local_subscriber_ids.json -TGID_URL: TGID_URL: http://downloads.freedmr.uk/downloads/talkgroup_ids.json -STALE_DAYS: 7 +TGID_URL: TGID_URL: https://freedmr.cymru/talkgroups/talkgroup_ids_json.php +STALE_DAYS: 1 LOCAL_SUBSCRIBER_FILE: local_subcriber_ids.json SUB_MAP_FILE: sub_map.pkl From 7903edce68eaf462ef181cb6c4540eb62a446028 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Jul 2022 01:15:52 +0100 Subject: [PATCH 03/10] Explicitly send NAK when we timeout a peer --- hblink.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hblink.py b/hblink.py index 2fed0cc..7e15032 100755 --- a/hblink.py +++ b/hblink.py @@ -769,6 +769,8 @@ class HBSYSTEM(DatagramProtocol): remove_list.append(peer) for peer in remove_list: logger.info('(%s) Peer %s (%s) has timed out and is being removed', self._system, self._peers[peer]['CALLSIGN'], self._peers[peer]['RADIO_ID']) + #First, NAK the peer + self.transport.write(b''.join([MSTNAK, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) # Remove any timed out peers from the configuration del self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer] if 'PEERS' not in self._CONFIG['SYSTEMS'][self._system] and 'OPTIONS' in self._CONFIG['SYSTEMS'][self._system]: From c25c6f0cc09750db4732957b54f37a85bb113a26 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Jul 2022 10:34:02 +0100 Subject: [PATCH 04/10] Send 3 x MSTCL from proxy --- hblink.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hblink.py b/hblink.py index 7e15032..c1770b4 100755 --- a/hblink.py +++ b/hblink.py @@ -769,8 +769,10 @@ class HBSYSTEM(DatagramProtocol): remove_list.append(peer) for peer in remove_list: logger.info('(%s) Peer %s (%s) has timed out and is being removed', self._system, self._peers[peer]['CALLSIGN'], self._peers[peer]['RADIO_ID']) - #First, NAK the peer - self.transport.write(b''.join([MSTNAK, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) + #First, MSTCL the peer - three times + self.transport.write(b''.join([MSTCL, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) + self.transport.write(b''.join([MSTCL, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) + self.transport.write(b''.join([MSTCL, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) # Remove any timed out peers from the configuration del self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer] if 'PEERS' not in self._CONFIG['SYSTEMS'][self._system] and 'OPTIONS' in self._CONFIG['SYSTEMS'][self._system]: From 7d25fb1cf9e07bfa9b4d98fde6c51babda4f4005 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Jul 2022 11:13:47 +0100 Subject: [PATCH 05/10] fixup --- hotspot_proxy_v2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hotspot_proxy_v2.py b/hotspot_proxy_v2.py index 6a8aa4e..84d5bb9 100644 --- a/hotspot_proxy_v2.py +++ b/hotspot_proxy_v2.py @@ -71,6 +71,10 @@ class Proxy(DatagramProtocol): if self.clientinfo and _peer_id != b'\xff\xff\xff\xff': print(f"{datetime.now().replace(microsecond=0)} Client: ID:{str(int_id(_peer_id)).rjust(9)} IP:{self.peerTrack[_peer_id]['shost'].rjust(15)} Port:{self.peerTrack[_peer_id]['sport']} Removed.") self.transport.write(b'RPTCL'+_peer_id, (self.master,self.peerTrack[_peer_id]['dport'])) + #Tell client we have closed do the session - 3 times, in case they are on a lossy network + self.transport.write(b'MSTCL',(self.peerTrack[_peer_id]['shost'],self.peerTrack[_peer_id]['sport'])) + self.transport.write(b'MSTCL',(self.peerTrack[_peer_id]['shost'],self.peerTrack[_peer_id]['sport'])) + self.transport.write(b'MSTCL',(self.peerTrack[_peer_id]['shost'],self.peerTrack[_peer_id]['sport'])) self.connTrack[self.peerTrack[_peer_id]['dport']] = False del self.peerTrack[_peer_id] From ffdcd019e408c54e942a369454bf53fe4e1f2b0f Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Jul 2022 11:30:26 +0100 Subject: [PATCH 06/10] Fix stack trace - key not found --- bridge_master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridge_master.py b/bridge_master.py index bd5a130..6c9e703 100644 --- a/bridge_master.py +++ b/bridge_master.py @@ -2621,7 +2621,7 @@ class routerHBP(HBSYSTEM): self.STATUS[_slot]['LOOPLOG'] = True self.STATUS[_slot]['LAST'] = pkt_time - if CONFIG['SYSTEMS'][self._system]['ENHANCED_OBP'] and '_bcsq' not in self.STATUS[_slot]: + if 'ENHANCED_OBP' in CONFIG['SYSTEMS'][self._system] and CONFIG['SYSTEMS'][self._system]['ENHANCED_OBP'] and '_bcsq' not in self.STATUS[_slot]: systems[self._system].send_bcsq(_dst_id,_stream_id) self.STATUS[_slot]['_bcsq'] = True return From 4423296f755f1669e99bf664b5e7150c75ef8592 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Jul 2022 21:31:12 +0100 Subject: [PATCH 07/10] 5s is plenty to wait for other side to respond to closedown - 15 sec was too much --- hotspot_proxy_v2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hotspot_proxy_v2.py b/hotspot_proxy_v2.py index 84d5bb9..deb59a5 100644 --- a/hotspot_proxy_v2.py +++ b/hotspot_proxy_v2.py @@ -71,7 +71,7 @@ class Proxy(DatagramProtocol): if self.clientinfo and _peer_id != b'\xff\xff\xff\xff': print(f"{datetime.now().replace(microsecond=0)} Client: ID:{str(int_id(_peer_id)).rjust(9)} IP:{self.peerTrack[_peer_id]['shost'].rjust(15)} Port:{self.peerTrack[_peer_id]['sport']} Removed.") self.transport.write(b'RPTCL'+_peer_id, (self.master,self.peerTrack[_peer_id]['dport'])) - #Tell client we have closed do the session - 3 times, in case they are on a lossy network + #Tell client we have closed the session - 3 times, in case they are on a lossy network self.transport.write(b'MSTCL',(self.peerTrack[_peer_id]['shost'],self.peerTrack[_peer_id]['sport'])) self.transport.write(b'MSTCL',(self.peerTrack[_peer_id]['shost'],self.peerTrack[_peer_id]['sport'])) self.transport.write(b'MSTCL',(self.peerTrack[_peer_id]['shost'],self.peerTrack[_peer_id]['sport'])) @@ -154,7 +154,7 @@ class Proxy(DatagramProtocol): # Remove the client after send a MSTN or MSTC packet if _command in (MSTN,MSTC): # Give time to the client for a reply to prevent port reassignment - self.peerTrack[_peer_id]['timer'].reset(15) + self.peerTrack[_peer_id]['timer'].reset(5) return From dc91a31cd10d8ee7f70eb80f6c93f4d15a84fde5 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Jul 2022 22:49:14 +0100 Subject: [PATCH 08/10] Maybe 15 was ok --- hotspot_proxy_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hotspot_proxy_v2.py b/hotspot_proxy_v2.py index deb59a5..d1e1efb 100644 --- a/hotspot_proxy_v2.py +++ b/hotspot_proxy_v2.py @@ -154,7 +154,7 @@ class Proxy(DatagramProtocol): # Remove the client after send a MSTN or MSTC packet if _command in (MSTN,MSTC): # Give time to the client for a reply to prevent port reassignment - self.peerTrack[_peer_id]['timer'].reset(5) + self.peerTrack[_peer_id]['timer'].reset(15) return From edbb3d420f2906638e675eae4d62f98ca2c92383 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 18 Jul 2022 02:18:44 +0100 Subject: [PATCH 09/10] Don't clobber files if they can't be downloaded --- hblink.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/hblink.py b/hblink.py index c1770b4..053eea5 100755 --- a/hblink.py +++ b/hblink.py @@ -769,9 +769,7 @@ class HBSYSTEM(DatagramProtocol): remove_list.append(peer) for peer in remove_list: logger.info('(%s) Peer %s (%s) has timed out and is being removed', self._system, self._peers[peer]['CALLSIGN'], self._peers[peer]['RADIO_ID']) - #First, MSTCL the peer - three times - self.transport.write(b''.join([MSTCL, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) - self.transport.write(b''.join([MSTCL, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) + #First, MSTCL the peer self.transport.write(b''.join([MSTCL, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) # Remove any timed out peers from the configuration del self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer] @@ -1389,12 +1387,13 @@ def try_download(_path, _file, _url, _stale,): result = 'ID ALIAS MAPPER: \'{}\' successfully downloaded'.format(_file) except IOError: result = 'ID ALIAS MAPPER: \'{}\' could not be downloaded due to an IOError'.format(_file) - try: - with open(_path+_file, 'wb') as outfile: - outfile.write(data) - outfile.close() - except IOError: - result = 'ID ALIAS mapper \'{}\' file could not be written due to an IOError'.format(_file) + else: + try: + with open(_path+_file, 'wb') as outfile: + outfile.write(data) + outfile.close() + except IOError: + result = 'ID ALIAS mapper \'{}\' file could not be written due to an IOError'.format(_file) else: result = 'ID ALIAS MAPPER: \'{}\' is current, not downloaded'.format(_file) From fef0774ff56799f4c8901fe81a28d7a7968d60c8 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 18 Jul 2022 22:00:46 +0100 Subject: [PATCH 10/10] Fix bug in master maintenence stopping peers from being removed on timeout --- hblink.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hblink.py b/hblink.py index 053eea5..d2a4a52 100755 --- a/hblink.py +++ b/hblink.py @@ -770,7 +770,7 @@ class HBSYSTEM(DatagramProtocol): for peer in remove_list: logger.info('(%s) Peer %s (%s) has timed out and is being removed', self._system, self._peers[peer]['CALLSIGN'], self._peers[peer]['RADIO_ID']) #First, MSTCL the peer - self.transport.write(b''.join([MSTCL, _peer_id]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) + self.transport.write(b''.join([MSTCL, peer]),self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer]['SOCKADDR']) # Remove any timed out peers from the configuration del self._CONFIG['SYSTEMS'][self._system]['PEERS'][peer] if 'PEERS' not in self._CONFIG['SYSTEMS'][self._system] and 'OPTIONS' in self._CONFIG['SYSTEMS'][self._system]: