From 2bf0ca8b6a5159ef24febbe75474de06b0bb1d4d Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 27 Dec 2022 22:23:54 +0000 Subject: [PATCH] Better checks on downnloaded json files, including reverting to backup files is required --- hblink.py | 46 ++++++------------------------ utils.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 37 deletions(-) create mode 100644 utils.py diff --git a/hblink.py b/hblink.py index 6f3eacc..7a0f1e4 100755 --- a/hblink.py +++ b/hblink.py @@ -45,7 +45,8 @@ from twisted.internet import reactor, task import log import config from const import * -from dmr_utils3.utils import int_id, bytes_4, mk_id_dict +from utils import mk_id_dict, try_download +from dmr_utils3.utils import int_id, bytes_4 # Imports for the reporting server import pickle @@ -1380,37 +1381,6 @@ class reportFactory(Factory): logger.debug('(REPORT) Send config') self.send_clients(b''.join([REPORT_OPCODES['CONFIG_SND'], serialized])) -#Use this try_download instead of that from dmr_utils3 -def try_download(_path, _file, _url, _stale,): - no_verify = ssl._create_unverified_context() - now = time() - file_exists = isfile(''.join([_path,_file])) == True - if file_exists: - file_old = (getmtime(''.join([_path,_file])) + _stale) < now - if not file_exists or (file_exists and file_old): - try: - with urlopen(_url, context=no_verify) as response: - data = response.read() - #outfile.write(data) - response.close() - result = 'ID ALIAS MAPPER: \'{}\' successfully downloaded'.format(_file) - except IOError: - result = 'ID ALIAS MAPPER: \'{}\' could not be downloaded due to an IOError'.format(_file) - else: - if data and (data != b'{}'): - try: - with open(''.join([_path,_file]), 'wb') as outfile: - outfile.write(data) - outfile.close() - except IOError: - result = 'ID ALIAS mapper \'{}\' file could not be written due to an IOError'.format(_file) - else: - result = 'ID ALIAS mapper \'{}\' file not written because downloaded data is empty for some reason'.format(_file) - - else: - result = 'ID ALIAS MAPPER: \'{}\' is current, not downloaded'.format(_file) - - return result #Read list of listed servers from CSV (actually TSV) file def mk_server_dict(path,filename): @@ -1475,15 +1445,17 @@ def mk_aliases(_config): except Exception as f: logger.error('(ALIAS) ID ALIAS MAPPER: Tried backup subscriber_ids file, but couldn\'t load that either: %s',f) else: + + if subscriber_ids: + logger.info('(ALIAS) ID ALIAS MAPPER: subscriber_ids dictionary is available') + #Add special IDs to DB subscriber_ids[900999] = 'D-APRS' subscriber_ids[4294967295] = 'SC' - if subscriber_ids: - logger.info('(ALIAS) ID ALIAS MAPPER: subscriber_ids dictionary is available') - try: - shutil.copy(_config['ALIASES']['PATH'] + _config['ALIASES']['SUBSCRIBER_FILE'],_config['ALIASES']['PATH'] + _config['ALIASES']['SUBSCRIBER_FILE'] + '.bak') - except IOError as g: + try: + shutil.copy(_config['ALIASES']['PATH'] + _config['ALIASES']['SUBSCRIBER_FILE'],_config['ALIASES']['PATH'] + _config['ALIASES']['SUBSCRIBER_FILE'] + '.bak') + except IOError as g: logger.info('(ALIAS) ID ALIAS MAPPER: couldn\'t make backup copy of subscriber_ids file %s',g) try: diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..2e7bf7c --- /dev/null +++ b/utils.py @@ -0,0 +1,84 @@ +# +############################################################################### +# Copyright (C) 2020 Simon Adlem, G7RZU +# Copyright (C) 2016-2019 Cortney T. Buffington, N0MJS +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +############################################################################### + +#Some utilty functions from dmr_utils3 have been modified. These live here. + +# Also new FreeDMR specific functions. + +import ssl +from time import time +from os.path import isfile, getmtime +from urllib.request import urlopen +from json import load as jload + + + +#Use this try_download instead of that from dmr_utils3 +def try_download(_path, _file, _url, _stale,): + no_verify = ssl._create_unverified_context() + now = time() + file_exists = isfile(''.join([_path,_file])) == True + if file_exists: + file_old = (getmtime(''.join([_path,_file])) + _stale) < now + if not file_exists or (file_exists and file_old): + try: + with urlopen(_url, context=no_verify) as response: + data = response.read() + #outfile.write(data) + response.close() + result = 'ID ALIAS MAPPER: \'{}\' successfully downloaded'.format(_file) + except IOError: + result = 'ID ALIAS MAPPER: \'{}\' could not be downloaded due to an IOError'.format(_file) + else: + if data and (data != b'{}'): + try: + with open(''.join([_path,_file]), 'wb') as outfile: + outfile.write(data) + outfile.close() + except IOError: + result = 'ID ALIAS mapper \'{}\' file could not be written due to an IOError'.format(_file) + else: + result = 'ID ALIAS mapper \'{}\' file not written because downloaded data is empty for some reason'.format(_file) + + else: + result = 'ID ALIAS MAPPER: \'{}\' is current, not downloaded'.format(_file) + + return result + +# SHORT VERSION - MAKES A SIMPLE {INTEGER ID: 'CALLSIGN'} DICTIONARY +def mk_id_dict(_path, _file): + _dict = {} + try: + with open(_path+_file, 'r', encoding='latin1') as _handle: + records = jload(_handle) + if 'count' in [*records]: + records.pop('count') + records = records[[*records][0]] + _handle.close + for record in records: + try: + _dict[int(record['id'])] = record['callsign'] + except: + pass + return _dict + except: + raise + +