From 8842e8b834ae8fc4dba3962109bae15ac655eda9 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Thu, 12 Sep 2024 19:59:00 +0200 Subject: [PATCH 01/23] Rename XLXHostFileDownloader => HostFileDownloader #50 --- ...XLXHostsFileDownloader.cpp => HostsFileDownloader.cpp} | 8 ++++---- .../{XLXHostsFileDownloader.h => HostsFileDownloader.h} | 2 +- DStarGateway/DStarGatewayApp.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename Common/{XLXHostsFileDownloader.cpp => HostsFileDownloader.cpp} (85%) rename Common/{XLXHostsFileDownloader.h => HostsFileDownloader.h} (96%) diff --git a/Common/XLXHostsFileDownloader.cpp b/Common/HostsFileDownloader.cpp similarity index 85% rename from Common/XLXHostsFileDownloader.cpp rename to Common/HostsFileDownloader.cpp index 0209378..12062c8 100644 --- a/Common/XLXHostsFileDownloader.cpp +++ b/Common/HostsFileDownloader.cpp @@ -20,17 +20,17 @@ #include #include -#include "XLXHostsFileDownloader.h" +#include "HostsFileDownloader.h" #include "Log.h" -size_t CXLXHostsFileDownloader::write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) +size_t CHostsFileDownloader::write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) { size_t written = fwrite(ptr, size, nmemb, stream); return written; } -/* wxHTTP randomly crashes when called on a worker thread, this must be called from main thread ! */ -std::string CXLXHostsFileDownloader::download(const std::string & xlxHostsFileURL) + +std::string CHostsFileDownloader::download(const std::string & xlxHostsFileURL) { CURL *curl; FILE *fp; diff --git a/Common/XLXHostsFileDownloader.h b/Common/HostsFileDownloader.h similarity index 96% rename from Common/XLXHostsFileDownloader.h rename to Common/HostsFileDownloader.h index 5c43a05..3f26182 100644 --- a/Common/XLXHostsFileDownloader.h +++ b/Common/HostsFileDownloader.h @@ -21,7 +21,7 @@ #include -class CXLXHostsFileDownloader { +class CHostsFileDownloader { public: static std::string download(const std::string & xlxHostsFileURL); private: diff --git a/DStarGateway/DStarGatewayApp.cpp b/DStarGateway/DStarGatewayApp.cpp index a62550b..b2cb51a 100644 --- a/DStarGateway/DStarGatewayApp.cpp +++ b/DStarGateway/DStarGatewayApp.cpp @@ -343,7 +343,7 @@ bool CDStarGatewayApp::createThread() TXLX xlxConfig; m_config->getXLX(xlxConfig); CLog::logInfo("XLX enabled: %d, Hosts file url: %s", int(xlxConfig.enabled), xlxConfig.url.c_str()); - m_thread->setXLX(xlxConfig.enabled, xlxConfig.enabled ? CXLXHostsFileDownloader::download(xlxConfig.url) : ""); + m_thread->setXLX(xlxConfig.enabled, xlxConfig.enabled ? CHostsFileDownloader::download(xlxConfig.url) : ""); // Setup Remote TRemote remoteConfig; From 20b434b8dd83ee2b510882785a96b89ad99513b8 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Thu, 12 Sep 2024 20:06:58 +0200 Subject: [PATCH 02/23] Fix wrong include #50 --- DStarGateway/DStarGatewayApp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DStarGateway/DStarGatewayApp.cpp b/DStarGateway/DStarGatewayApp.cpp index b2cb51a..9cf45c5 100644 --- a/DStarGateway/DStarGatewayApp.cpp +++ b/DStarGateway/DStarGatewayApp.cpp @@ -41,7 +41,7 @@ #include "Version.h" #include "GitVersion.h" #include "RepeaterProtocolHandlerFactory.h" -#include "XLXHostsFileDownloader.h" +#include "HostsFileDownloader.h" #include "Log.h" #include "LogFileTarget.h" #include "LogConsoleTarget.h" From 7fb469e67f4e33ab752c968f570f0bf42f476a3d Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Thu, 12 Sep 2024 20:36:19 +0200 Subject: [PATCH 03/23] Change default build task #50 --- .vscode/tasks.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9f00615..bf6c999 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -26,7 +26,10 @@ "USE_GPSD=1", "DStarGateway/dstargateway" ], - "group": "build", + "group": { + "kind": "build", + "isDefault": true + }, "problemMatcher": [] }, { @@ -91,10 +94,7 @@ "ENABLE_DEBUG=1", "USE_GPSD=1" ], - "group": { - "kind": "build", - "isDefault": true - }, + "group": "build", "problemMatcher": [] } ] From 4611ac66b0b10019b655d8ad528336916a90cfc9 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Thu, 12 Sep 2024 21:18:51 +0200 Subject: [PATCH 04/23] Download all host files on startup #50 --- Common/Defs.h | 1 + Common/HostsFileDownloader.cpp | 19 ++++++++++++------- Common/HostsFileDownloader.h | 2 +- DStarGateway/DStarGatewayApp.cpp | 16 ++++++++++------ DStarGateway/DStarGatewayConfig.cpp | 7 +++++-- DStarGateway/DStarGatewayConfig.h | 5 ++++- DStarGateway/DStarGatewayThread.cpp | 7 +++---- DStarGateway/DStarGatewayThread.h | 3 +-- DStarGateway/example.cfg | 9 +++++++-- 9 files changed, 44 insertions(+), 25 deletions(-) diff --git a/Common/Defs.h b/Common/Defs.h index 2f12568..9cb2d07 100644 --- a/Common/Defs.h +++ b/Common/Defs.h @@ -25,6 +25,7 @@ const std::string DEXTRA_HOSTS_FILE_NAME("DExtra_Hosts.txt"); const std::string DCS_HOSTS_FILE_NAME("DCS_Hosts.txt"); const std::string DPLUS_HOSTS_FILE_NAME("DPlus_Hosts.txt"); +const std::string XLX_HOSTS_FILE_NAME("XLX_Hosts.txt"); const std::string GATEWAY_HOSTS_FILE_NAME("Gateway_Hosts.txt"); const std::string LINKS_BASE_NAME("Links"); diff --git a/Common/HostsFileDownloader.cpp b/Common/HostsFileDownloader.cpp index 12062c8..74fdb29 100644 --- a/Common/HostsFileDownloader.cpp +++ b/Common/HostsFileDownloader.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include "HostsFileDownloader.h" @@ -30,21 +31,21 @@ size_t CHostsFileDownloader::write_data(void *ptr, size_t size, size_t nmemb, FI } -std::string CHostsFileDownloader::download(const std::string & xlxHostsFileURL) +bool CHostsFileDownloader::download(const std::string & hostsFileURL, const std::string & hostFilePath) { CURL *curl; FILE *fp; bool ok = false; std::string outFileName; - char outFileNameBuf[] = "/tmp/XLXHostFile_XXXXXX"; + char outFileNameBuf[] = "/tmp/HostFile_XXXXXX"; - CLog::logInfo("Downloading XLX host file from %s", xlxHostsFileURL.c_str()); + CLog::logInfo("Downloading host file from %s", hostsFileURL.c_str()); curl = curl_easy_init(); if (curl) { int filedes = mkstemp(outFileNameBuf); - if(filedes > 0 && (fp = fdopen(filedes,"wb")) != NULL) { - curl_easy_setopt(curl, CURLOPT_URL, xlxHostsFileURL.c_str()); + if(filedes > 0 && (fp = fdopen(filedes,"wb")) != nullptr) { + curl_easy_setopt(curl, CURLOPT_URL, hostsFileURL.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); CURLcode res = curl_easy_perform(curl); @@ -58,9 +59,13 @@ std::string CHostsFileDownloader::download(const std::string & xlxHostsFileURL) if(ok) { outFileName = std::string(outFileNameBuf); + ok = ok && std::filesystem::copy_file(outFileName, hostFilePath, std::filesystem::copy_options::overwrite_existing); + if(!ok) { + CLog::logError("Failed to copy host file to %s", hostFilePath.c_str()); + } } else { - CLog::logError("Failed to download XLx Host file from %s", xlxHostsFileURL.c_str()); + CLog::logWarning("Failed to download Host file from %s, using previous file", hostsFileURL.c_str()); } - return outFileName; + return ok; } diff --git a/Common/HostsFileDownloader.h b/Common/HostsFileDownloader.h index 3f26182..dab6e27 100644 --- a/Common/HostsFileDownloader.h +++ b/Common/HostsFileDownloader.h @@ -23,7 +23,7 @@ class CHostsFileDownloader { public: - static std::string download(const std::string & xlxHostsFileURL); + static bool download(const std::string & hostsFileURL, const std::string & hostFilePath); private: static size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream); }; diff --git a/DStarGateway/DStarGatewayApp.cpp b/DStarGateway/DStarGatewayApp.cpp index 9cf45c5..da8f50e 100644 --- a/DStarGateway/DStarGatewayApp.cpp +++ b/DStarGateway/DStarGatewayApp.cpp @@ -35,13 +35,13 @@ #include "DStarGatewayConfig.h" #include "DStarGatewayApp.h" #include "Version.h" +#include "HostsFileDownloader.h" #include "IRCDDBMultiClient.h" #include "IRCDDBClient.h" #include "Utils.h" #include "Version.h" #include "GitVersion.h" #include "RepeaterProtocolHandlerFactory.h" -#include "HostsFileDownloader.h" #include "Log.h" #include "LogFileTarget.h" #include "LogConsoleTarget.h" @@ -324,26 +324,30 @@ bool CDStarGatewayApp::createThread() // Setup Dextra TDextra dextraConfig; m_config->getDExtra(dextraConfig); - CLog::logInfo("DExtra enabled: %d, max. dongles: %u", int(dextraConfig.enabled), dextraConfig.maxDongles); + CLog::logInfo("DExtra enabled: %d, max. dongles: %u, url: %s", int(dextraConfig.enabled), dextraConfig.maxDongles, dextraConfig.hostfileUrl.c_str()); + if(dextraConfig.enabled) CHostsFileDownloader::download(dextraConfig.hostfileUrl, paths.dataDir + "/" + DEXTRA_HOSTS_FILE_NAME); m_thread->setDExtra(dextraConfig.enabled, dextraConfig.maxDongles); // Setup DCS TDCS dcsConfig; m_config->getDCS(dcsConfig); - CLog::logInfo("DCS enabled: %d", int(dcsConfig.enabled)); + CLog::logInfo("DCS enabled: %d, url: %s", int(dcsConfig.enabled), dcsConfig.hostfileUrl.c_str()); + if(dextraConfig.enabled) CHostsFileDownloader::download(dcsConfig.hostfileUrl, paths.dataDir + "/" + DCS_HOSTS_FILE_NAME); m_thread->setDCS(dcsConfig.enabled); // Setup DPlus TDplus dplusConfig; m_config->getDPlus(dplusConfig); - CLog::logInfo("D-Plus enabled: %d, max. dongles: %u, login: %s", int(dplusConfig.enabled), dplusConfig.maxDongles, dplusConfig.login.c_str()); + CLog::logInfo("D-Plus enabled: %d, max. dongles: %u, login: %s, url: %s", int(dplusConfig.enabled), dplusConfig.maxDongles, dplusConfig.login.c_str(), dplusConfig.hostfileUrl.c_str()); + if(dplusConfig.enabled) CHostsFileDownloader::download(dplusConfig.hostfileUrl, paths.dataDir + "/" + DPLUS_HOSTS_FILE_NAME); m_thread->setDPlus(dplusConfig.enabled, dplusConfig.maxDongles, dplusConfig.login); // Setup XLX TXLX xlxConfig; m_config->getXLX(xlxConfig); - CLog::logInfo("XLX enabled: %d, Hosts file url: %s", int(xlxConfig.enabled), xlxConfig.url.c_str()); - m_thread->setXLX(xlxConfig.enabled, xlxConfig.enabled ? CHostsFileDownloader::download(xlxConfig.url) : ""); + CLog::logInfo("XLX enabled: %d, Hosts file url: %s", int(xlxConfig.enabled), xlxConfig.hostfileUrl.c_str()); + if(xlxConfig.enabled) CHostsFileDownloader::download(xlxConfig.hostfileUrl, paths.dataDir + "/" + XLX_HOSTS_FILE_NAME); + m_thread->setXLX(xlxConfig.enabled); // Setup Remote TRemote remoteConfig; diff --git a/DStarGateway/DStarGatewayConfig.cpp b/DStarGateway/DStarGatewayConfig.cpp index da334be..eade624 100644 --- a/DStarGateway/DStarGatewayConfig.cpp +++ b/DStarGateway/DStarGatewayConfig.cpp @@ -83,9 +83,9 @@ bool CDStarGatewayConfig::loadDaemon(const CConfig & cfg) bool CDStarGatewayConfig::loadXLX(const CConfig & cfg) { bool ret = cfg.getValue("xlx", "enabled", m_xlx.enabled, true); - ret = cfg.getValue("xlx", "hostfileUrl", m_xlx.url, 0, 1024, "") && ret; + ret = cfg.getValue("xlx", "hostfileUrl", m_xlx.hostfileUrl, 0, 1024, "") && ret; - m_xlx.enabled = m_xlx.enabled && !m_xlx.url.empty(); + m_xlx.enabled = m_xlx.enabled; return ret; } @@ -105,6 +105,7 @@ bool CDStarGatewayConfig::loadDextra(const CConfig & cfg) { bool ret = cfg.getValue("dextra", "enabled", m_dextra.enabled, true); ret = cfg.getValue("dextra", "maxDongles", m_dextra.maxDongles, 1U, 5U, 5U) && ret; + ret = cfg.getValue("dextra", "hostfileUrl", m_dextra.hostfileUrl, 0, 1024, "") && ret; return ret; } @@ -113,6 +114,7 @@ bool CDStarGatewayConfig::loadDPlus(const CConfig & cfg) bool ret = cfg.getValue("dplus", "enabled", m_dplus.enabled, true); ret = cfg.getValue("dplus", "maxDongles", m_dplus.maxDongles, 1U, 5U, 5U) && ret; ret = cfg.getValue("dplus", "login", m_dplus.login, 0, LONG_CALLSIGN_LENGTH, m_gateway.callsign) && ret; + ret = cfg.getValue("dplus", "hostfileUrl", m_dplus.hostfileUrl, 0, 1024, "") && ret; m_dplus.enabled = m_dplus.enabled && !m_dplus.login.empty(); m_dplus.login = CUtils::ToUpper(m_dplus.login); @@ -123,6 +125,7 @@ bool CDStarGatewayConfig::loadDPlus(const CConfig & cfg) bool CDStarGatewayConfig::loadDCS(const CConfig & cfg) { bool ret = cfg.getValue("dcs", "enabled", m_dcs.enabled, true); + ret = cfg.getValue("dcs", "hostfileUrl", m_dcs.hostfileUrl, 0, 1024, "") && ret; return ret; } diff --git a/DStarGateway/DStarGatewayConfig.h b/DStarGateway/DStarGatewayConfig.h index 1a15d65..016df97 100644 --- a/DStarGateway/DStarGatewayConfig.h +++ b/DStarGateway/DStarGatewayConfig.h @@ -102,16 +102,19 @@ typedef struct { typedef struct { bool enabled; unsigned int maxDongles; + std::string hostfileUrl; } TDextra; typedef struct { bool enabled; std::string login; unsigned int maxDongles; + std::string hostfileUrl; } TDplus; typedef struct { bool enabled; + std::string hostfileUrl; } TDCS; typedef struct { @@ -120,7 +123,7 @@ typedef struct { typedef struct { bool enabled; - std::string url; + std::string hostfileUrl; } TXLX; typedef struct { diff --git a/DStarGateway/DStarGatewayThread.cpp b/DStarGateway/DStarGatewayThread.cpp index 3f4c218..ac8cbe8 100644 --- a/DStarGateway/DStarGatewayThread.cpp +++ b/DStarGateway/DStarGatewayThread.cpp @@ -88,7 +88,6 @@ m_dplusMaxDongles(0U), m_dplusLogin(), m_dcsEnabled(true), m_xlxEnabled(true), -m_xlxHostsFileName(), m_ccsEnabled(true), m_ccsHost(), m_infoEnabled(true), @@ -599,10 +598,9 @@ void CDStarGatewayThread::setDCS(bool enabled) m_dcsEnabled = enabled; } -void CDStarGatewayThread::setXLX(bool enabled, const std::string& xlxHostsFileName) +void CDStarGatewayThread::setXLX(bool enabled) { m_xlxEnabled = enabled; - m_xlxHostsFileName = xlxHostsFileName; } #ifdef USE_CCS @@ -1150,7 +1148,8 @@ void CDStarGatewayThread::loadGateways() void CDStarGatewayThread::loadAllReflectors() { if (m_xlxEnabled) { - loadReflectors(m_xlxHostsFileName, DP_DCS); + std::string fileName = m_dataDir + "/" + DPLUS_HOSTS_FILE_NAME; + loadReflectors(fileName, DP_DCS); } if (m_dplusEnabled) { diff --git a/DStarGateway/DStarGatewayThread.h b/DStarGateway/DStarGatewayThread.h index 26a799b..20e6664 100644 --- a/DStarGateway/DStarGatewayThread.h +++ b/DStarGateway/DStarGatewayThread.h @@ -61,7 +61,7 @@ public: virtual void setDExtra(bool enabled, unsigned int maxDongles); virtual void setDPlus(bool enabled, unsigned int maxDongles, const std::string& login); virtual void setDCS(bool enabled); - virtual void setXLX(bool enabled, const std::string& fileName); + virtual void setXLX(bool enabled); #ifdef USE_CCS virtual void setCCS(bool enabled, const std::string& host); #endif @@ -112,7 +112,6 @@ private: std::string m_dplusLogin; bool m_dcsEnabled; bool m_xlxEnabled; - std::string m_xlxHostsFileName; bool m_ccsEnabled; std::string m_ccsHost; bool m_infoEnabled; diff --git a/DStarGateway/example.cfg b/DStarGateway/example.cfg index 73d7b09..23ae692 100644 --- a/DStarGateway/example.cfg +++ b/DStarGateway/example.cfg @@ -155,23 +155,28 @@ displayLevel= # defaults to info, valid values are trace, debug, info, warning repeatThreshold=#defaults to 2, valid values are disbaled and 1 to 10. Prevents flooding of logs from repeated log messages. [Paths] -data=/usr/local/share/dstargateway.d/ #Path where the data (hostfiles, audio files etc) can be found +data=/usr/local/share/dstargateway.d/ # Path where the data (auto downbloaded hostfiles, audio files etc) can be found +customHostfiles=/usr/local/share/dstargateway.d/hostfiles.d/ # Place your custom host files in this directory, this dir must be different from datadir. + # Any hosts found here will override same host in downloaded files [DExtra] enabled=true # There is no reason to disable this maxDongles=5 +hostfileUrl=http://www.pistar.uk/downloads/DExtra_Hosts.txt # URL where to download the host file on startup and every X minutes. Leave empty to disable. [DPlus] enabled=true # There is no reason to disable this maxDongles=5 login= # defaults to gateway callsign +hostfileUrl=http://www.pistar.uk/downloads/DPlus_Hosts.txt # URL where to download the host file on startup and every X minutes. Leave empty to disable. [DCS] enabled=true # There is no reason to disable this +hostfileUrl=http://www.pistar.uk/downloads/DCS_Hosts.txt # URL where to download the host file on startup and every X minutes. Leave empty to disable. [XLX] enabled=true -hostfileUrl=http://xlxapi.rlx.lu/api.php?do=GetXLXDMRMaster +hostfileUrl=http://xlxapi.rlx.lu/api.php?do=GetXLXDMRMaster # URL where to download the host file on startup and every X minutes. Leave empty to disable. [DRats] enabled=false # Defaults to false. The program need to be compiled with DRats support for DRats to be actually enabled From 74884d9cc3db07f00995f91c5a7bd15e04606191 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Fri, 13 Sep 2024 19:59:44 +0200 Subject: [PATCH 05/23] update log messages #50 --- DStarGateway/DStarGatewayThread.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DStarGateway/DStarGatewayThread.cpp b/DStarGateway/DStarGatewayThread.cpp index ac8cbe8..def3e12 100644 --- a/DStarGateway/DStarGatewayThread.cpp +++ b/DStarGateway/DStarGatewayThread.cpp @@ -405,16 +405,16 @@ void* CDStarGatewayThread::Entry() } catch (std::exception& e) { std::string message(e.what()); - CLog::logFatal("Exception raised in the main thread - \"%s\"", message.c_str()); + CLog::logFatal("Exception raised in the DStar Gateway thread - \"%s\"", message.c_str()); throw; } catch (...) { - CLog::logFatal("Unknown exception raised in the main thread"); + CLog::logFatal("Unknown exception raised in the DStar Gateway thread"); throw; } #endif - CLog::logInfo("Stopping the ircDDB Gateway thread"); + CLog::logInfo("Stopping the DStar Gateway thread"); // Unlink from all reflectors CDExtraHandler::unlink(); From c8dc79ce5f388747ad51a1c89cc0163da06deebf Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Mon, 16 Sep 2024 17:31:26 +0200 Subject: [PATCH 06/23] rename constant #50 --- DStarGateway/DStarGatewayThread.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DStarGateway/DStarGatewayThread.h b/DStarGateway/DStarGatewayThread.h index 20e6664..f279b6f 100644 --- a/DStarGateway/DStarGatewayThread.h +++ b/DStarGateway/DStarGatewayThread.h @@ -17,8 +17,8 @@ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#ifndef IRCDDBGatewayThread_H -#define IRCDDBGatewayThread_H +#ifndef DStarGatewayThread_H +#define DStarGatewayThread_H #include "DummyRepeaterProtocolHandler.h" #include "IcomRepeaterProtocolHandler.h" From 707f9e754c0303fa521439d78894ac909a0fff9c Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Mon, 16 Sep 2024 19:56:15 +0200 Subject: [PATCH 07/23] move hosts files logic to seprate class, add custom host files #50 --- DStarGateway/DStarGatewayApp.cpp | 13 +- DStarGateway/DStarGatewayConfig.cpp | 5 + DStarGateway/DStarGatewayConfig.h | 1 + DStarGateway/DStarGatewayThread.cpp | 84 +----------- DStarGateway/DStarGatewayThread.h | 8 -- DStarGateway/HostsFilesManager.cpp | 203 ++++++++++++++++++++++++++++ DStarGateway/HostsFilesManager.h | 65 +++++++++ 7 files changed, 287 insertions(+), 92 deletions(-) create mode 100644 DStarGateway/HostsFilesManager.cpp create mode 100644 DStarGateway/HostsFilesManager.h diff --git a/DStarGateway/DStarGatewayApp.cpp b/DStarGateway/DStarGatewayApp.cpp index da8f50e..7ee437e 100644 --- a/DStarGateway/DStarGatewayApp.cpp +++ b/DStarGateway/DStarGatewayApp.cpp @@ -35,7 +35,6 @@ #include "DStarGatewayConfig.h" #include "DStarGatewayApp.h" #include "Version.h" -#include "HostsFileDownloader.h" #include "IRCDDBMultiClient.h" #include "IRCDDBClient.h" #include "Utils.h" @@ -50,6 +49,7 @@ #include "Daemon.h" #include "APRSISHandlerThread.h" #include "DummyAPRSHandlerThread.h" +#include "HostsFilesManager.h" CDStarGatewayApp * CDStarGatewayApp::g_app = nullptr; const std::string BANNER_1 = CStringUtils::string_format("%s Copyright (C) %s\n", FULL_PRODUCT_NAME.c_str(), VENDOR_NAME.c_str()); @@ -325,30 +325,33 @@ bool CDStarGatewayApp::createThread() TDextra dextraConfig; m_config->getDExtra(dextraConfig); CLog::logInfo("DExtra enabled: %d, max. dongles: %u, url: %s", int(dextraConfig.enabled), dextraConfig.maxDongles, dextraConfig.hostfileUrl.c_str()); - if(dextraConfig.enabled) CHostsFileDownloader::download(dextraConfig.hostfileUrl, paths.dataDir + "/" + DEXTRA_HOSTS_FILE_NAME); m_thread->setDExtra(dextraConfig.enabled, dextraConfig.maxDongles); // Setup DCS TDCS dcsConfig; m_config->getDCS(dcsConfig); CLog::logInfo("DCS enabled: %d, url: %s", int(dcsConfig.enabled), dcsConfig.hostfileUrl.c_str()); - if(dextraConfig.enabled) CHostsFileDownloader::download(dcsConfig.hostfileUrl, paths.dataDir + "/" + DCS_HOSTS_FILE_NAME); m_thread->setDCS(dcsConfig.enabled); // Setup DPlus TDplus dplusConfig; m_config->getDPlus(dplusConfig); CLog::logInfo("D-Plus enabled: %d, max. dongles: %u, login: %s, url: %s", int(dplusConfig.enabled), dplusConfig.maxDongles, dplusConfig.login.c_str(), dplusConfig.hostfileUrl.c_str()); - if(dplusConfig.enabled) CHostsFileDownloader::download(dplusConfig.hostfileUrl, paths.dataDir + "/" + DPLUS_HOSTS_FILE_NAME); m_thread->setDPlus(dplusConfig.enabled, dplusConfig.maxDongles, dplusConfig.login); // Setup XLX TXLX xlxConfig; m_config->getXLX(xlxConfig); CLog::logInfo("XLX enabled: %d, Hosts file url: %s", int(xlxConfig.enabled), xlxConfig.hostfileUrl.c_str()); - if(xlxConfig.enabled) CHostsFileDownloader::download(xlxConfig.hostfileUrl, paths.dataDir + "/" + XLX_HOSTS_FILE_NAME); m_thread->setXLX(xlxConfig.enabled); + // Setup hostsfiles + CHostsFilesManager::setHostFilesDirectories(paths.dataDir, paths.customHostsFiles); + CHostsFilesManager::setDextra(dextraConfig.enabled, dextraConfig.hostfileUrl); + CHostsFilesManager::setDCS (dcsConfig.enabled, dcsConfig.hostfileUrl); + CHostsFilesManager::setDPlus (dplusConfig.enabled, dplusConfig.hostfileUrl); + CHostsFilesManager::setXLX (xlxConfig.enabled, xlxConfig.hostfileUrl); + // Setup Remote TRemote remoteConfig; m_config->getRemote(remoteConfig); diff --git a/DStarGateway/DStarGatewayConfig.cpp b/DStarGateway/DStarGatewayConfig.cpp index eade624..2438e87 100644 --- a/DStarGateway/DStarGatewayConfig.cpp +++ b/DStarGateway/DStarGatewayConfig.cpp @@ -195,11 +195,16 @@ bool CDStarGatewayConfig::loadLog(const CConfig & cfg) bool CDStarGatewayConfig::loadPaths(const CConfig & cfg) { bool ret = cfg.getValue("paths", "data", m_paths.dataDir, 0, 2048, "/usr/local/share/dstargateway.d/"); + ret = cfg.getValue("customHostfiles", "data", m_paths.customHostsFiles, 0, 2048, "/usr/local/share/dstargateway.d/hostfiles.d/") && ret; if(ret && m_paths.dataDir[m_paths.dataDir.length() - 1] != '/') { m_paths.dataDir.push_back('/'); } + if(ret && m_paths.dataDir[m_paths.customHostsFiles.length() - 1] != '/') { + m_paths.customHostsFiles.push_back('/'); + } + //TODO 20211226 check if directory are accessible return ret; diff --git a/DStarGateway/DStarGatewayConfig.h b/DStarGateway/DStarGatewayConfig.h index 016df97..6a620e5 100644 --- a/DStarGateway/DStarGatewayConfig.h +++ b/DStarGateway/DStarGatewayConfig.h @@ -80,6 +80,7 @@ typedef struct { typedef struct { std::string dataDir; + std::string customHostsFiles; } Tpaths; typedef struct { diff --git a/DStarGateway/DStarGatewayThread.cpp b/DStarGateway/DStarGatewayThread.cpp index def3e12..0a65d90 100644 --- a/DStarGateway/DStarGatewayThread.cpp +++ b/DStarGateway/DStarGatewayThread.cpp @@ -54,6 +54,7 @@ #include "Defs.h" #include "Log.h" #include "StringUtils.h" +#include "HostsFilesManager.h" const std::string LOOPBACK_ADDRESS("127.0.0.1"); @@ -136,6 +137,9 @@ CDStarGatewayThread::~CDStarGatewayThread() void* CDStarGatewayThread::Entry() { + CHostsFilesManager::setCache(&m_cache); + auto hostFut = CHostsFilesManager::UpdateHostsAsync(); // Do this in a separate thread + // Truncate the old Links.log file std::string fullName = m_logDir + "/" + LINKS_BASE_NAME + ".log"; if (!m_name.empty()) { @@ -195,6 +199,7 @@ void* CDStarGatewayThread::Entry() } // Wait here until we have the essentials to run + hostFut.get(); while (!m_killed && (m_dextraPool == NULL || m_dplusPool == NULL || m_dcsPool == NULL || m_g2HandlerPool == NULL || (m_icomRepeaterHandler == NULL && m_hbRepeaterHandler == NULL && m_dummyRepeaterHandler == NULL) || m_gatewayCallsign.empty())) ::std::this_thread::sleep_for(std::chrono::milliseconds(500UL)); // 1/2 sec @@ -217,9 +222,6 @@ void* CDStarGatewayThread::Entry() } } - loadGateways(); - loadAllReflectors(); - CG2Handler::setG2ProtocolHandlerPool(m_g2HandlerPool); CG2Handler::setHeaderLogger(headerLogger); @@ -1139,82 +1141,6 @@ void CDStarGatewayThread::processDD() } } -void CDStarGatewayThread::loadGateways() -{ - std::string fileName = m_dataDir + "/" + GATEWAY_HOSTS_FILE_NAME; - loadReflectors(fileName, DP_DEXTRA); -} - -void CDStarGatewayThread::loadAllReflectors() -{ - if (m_xlxEnabled) { - std::string fileName = m_dataDir + "/" + DPLUS_HOSTS_FILE_NAME; - loadReflectors(fileName, DP_DCS); - } - - if (m_dplusEnabled) { - std::string fileName = m_dataDir + "/" + DPLUS_HOSTS_FILE_NAME; - loadReflectors(fileName, DP_DPLUS); - } - - if (m_dextraEnabled) { - std::string fileName = m_dataDir + "/" + DEXTRA_HOSTS_FILE_NAME; - loadReflectors(fileName, DP_DEXTRA); - } - - if (m_dcsEnabled) { - std::string fileName = m_dataDir + "/" + DCS_HOSTS_FILE_NAME; - loadReflectors(fileName, DP_DCS); - } -} - -void CDStarGatewayThread::loadReflectors(std::string hostFileName, DSTAR_PROTOCOL proto) -{ - unsigned int count = 0U; - - CHostFile hostFile(hostFileName, false); - for (unsigned int i = 0U; i < hostFile.getCount(); i++) { - std::string reflector = hostFile.getName(i); - in_addr address = CUDPReaderWriter::lookup(hostFile.getAddress(i)); - bool lock = hostFile.getLock(i); - - if (address.s_addr != INADDR_NONE) { - unsigned char* ucp = (unsigned char*)&address; - - std::string addrText; - addrText = CStringUtils::string_format("%u.%u.%u.%u", ucp[0U] & 0xFFU, ucp[1U] & 0xFFU, ucp[2U] & 0xFFU, ucp[3U] & 0xFFU); - - if (lock) - CLog::logInfo("Locking %s to %s", reflector.c_str(), addrText.c_str()); - - reflector.resize(LONG_CALLSIGN_LENGTH - 1U, ' '); - reflector += "G"; - m_cache.updateGateway(reflector, addrText, proto, lock, true); - - count++; - } - } - - std::string protoString; - switch (proto) - { - case DP_DEXTRA: - protoString = "DExtra"; - break; - case DP_DCS: - protoString = "DCS"; - break; - case DP_DPLUS: - protoString = "DPlus"; - break; - default: - // ??? - break; - } - - CLog::logInfo("Loaded %u of %u %s hosts from %s", count, hostFile.getCount(), protoString.c_str() , hostFileName.c_str()); -} - void CDStarGatewayThread::writeStatus() { std::string fullName = LINKS_BASE_NAME; diff --git a/DStarGateway/DStarGatewayThread.h b/DStarGateway/DStarGatewayThread.h index f279b6f..68b9591 100644 --- a/DStarGateway/DStarGatewayThread.h +++ b/DStarGateway/DStarGatewayThread.h @@ -146,14 +146,6 @@ private: void processG2(); void processDD(); - void loadGateways(); - void loadAllReflectors(); - void loadReflectors(std::string hostFileName, DSTAR_PROTOCOL proto); - void loadDExtraReflectors(const std::string& fileName); - void loadDPlusReflectors(const std::string& fileName); - void loadDCSReflectors(const std::string& fileName); - void loadXLXReflectors(); - void writeStatus(); void readStatusFiles(); diff --git a/DStarGateway/HostsFilesManager.cpp b/DStarGateway/HostsFilesManager.cpp new file mode 100644 index 0000000..60805d0 --- /dev/null +++ b/DStarGateway/HostsFilesManager.cpp @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2024 by Geoffrey F4FXL / KC3FRA + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include + +#include "HostsFilesManager.h" +#include "HostsFileDownloader.h" +#include "HostFile.h" +#include "StringUtils.h" +#include "UDPReaderWriter.h" +#include "Log.h" + +std::string CHostsFilesManager::m_hostFilesDirectory(""); +std::string CHostsFilesManager::m_customFilesDirectory(""); + +bool CHostsFilesManager::m_dextraEnabled = false; +bool CHostsFilesManager::m_dcsEnabled = false; +bool CHostsFilesManager::m_dplusEnabled = false; +bool CHostsFilesManager::m_xlxEnabled = false; + +std::string CHostsFilesManager::m_dextraUrl(""); +std::string CHostsFilesManager::m_dcsUrl(""); +std::string CHostsFilesManager::m_dplusUrl(""); +std::string CHostsFilesManager::m_xlxUrl(""); + +CCacheManager * CHostsFilesManager::m_cache = nullptr; +CTimer CHostsFilesManager::m_downloadTimer(1000U, 60 * 60 * 24); + +void CHostsFilesManager::setHostFilesDirectories(const std::string & hostFilesDir, const std::string & customHostFilesDir) +{ + m_hostFilesDirectory.assign(hostFilesDir); + m_customFilesDirectory.assign(customHostFilesDir); +} + +void CHostsFilesManager::setDextra(bool enabled, const std::string & url) +{ + m_dextraEnabled = enabled; + m_dextraUrl.assign(url); +} + +void CHostsFilesManager::setDPlus(bool enabled, const std::string & url) +{ + m_dplusEnabled = enabled; + m_dplusUrl.assign(url); +} + +void CHostsFilesManager::setDCS(bool enabled, const std::string & url) +{ + m_dcsEnabled = enabled; + m_dcsUrl.assign(url); +} + +void CHostsFilesManager::setXLX(bool enabled, const std::string & url) +{ + m_xlxEnabled = enabled; + m_xlxUrl.assign(url); +} + +void CHostsFilesManager::setCache(CCacheManager * cache) +{ + assert(cache != nullptr); + m_cache = cache; +} + +void CHostsFilesManager::clock(unsigned int ms) +{ + m_downloadTimer.clock(ms); + + if(m_downloadTimer.hasExpired()) { + UpdateHostsAsync(); // call and forget + } +} + +void CHostsFilesManager::setDownloadTimeout(unsigned int seconds) +{ + m_downloadTimer.setTimeout(seconds); +} + +bool CHostsFilesManager::UpdateHostsFromInternet() +{ + CLog::logInfo("Updating hosts files from internet"); + bool ret = true; + if(m_dextraEnabled && !m_dextraUrl.empty()) ret = CHostsFileDownloader::download(m_dextraUrl, m_hostFilesDirectory + "/" + DEXTRA_HOSTS_FILE_NAME) && ret; + if(m_dcsEnabled && !m_dcsUrl.empty()) ret = CHostsFileDownloader::download(m_dcsUrl, m_hostFilesDirectory + "/" + DCS_HOSTS_FILE_NAME) && ret; + if(m_dplusEnabled && !m_dplusUrl.empty()) ret = CHostsFileDownloader::download(m_dplusUrl, m_hostFilesDirectory + "/" + DPLUS_HOSTS_FILE_NAME) && ret; + if(m_xlxEnabled && !m_xlxUrl.empty()) ret = CHostsFileDownloader::download(m_xlxUrl, m_hostFilesDirectory + "/" + XLX_HOSTS_FILE_NAME) && ret; + + if(!ret) CLog::logWarning("Some hosts files failed to downlaod"); + + CHostsFilesManager::loadReflectors(m_hostFilesDirectory); + + return ret; +} + +bool CHostsFilesManager::UpdateHostsFromLocal() +{ + CHostsFilesManager::loadReflectors(m_customFilesDirectory); + return true; +} + +bool CHostsFilesManager::UpdateHosts() +{ + bool ret = UpdateHostsFromInternet(); + UpdateHostsFromLocal() && ret; + + return ret; +} + +std::future CHostsFilesManager::UpdateHostsAsync() +{ + auto fut = std::async(std::launch::async, UpdateHosts); + return fut; +} + +void CHostsFilesManager::loadReflectors(const std::string & directory) +{ + if (m_xlxEnabled) { + std::string fileName = directory + "/" + DPLUS_HOSTS_FILE_NAME; + loadReflectors(fileName, DP_DCS); + } + + if (m_dplusEnabled) { + std::string fileName = directory + "/" + DPLUS_HOSTS_FILE_NAME; + loadReflectors(fileName, DP_DPLUS); + } + + if (m_dextraEnabled) { + std::string fileName = directory + "/" + DEXTRA_HOSTS_FILE_NAME; + loadReflectors(fileName, DP_DEXTRA); + } + + if (m_dcsEnabled) { + std::string fileName = directory + "/" + DCS_HOSTS_FILE_NAME; + loadReflectors(fileName, DP_DCS); + } +} + +void CHostsFilesManager::loadReflectors(const std::string & hostFileName, DSTAR_PROTOCOL proto) +{ + if(m_cache == nullptr) { + CLog::logDebug("HostsFilesManager cache not initilized"); + return; + } + + unsigned int count = 0U; + + CHostFile hostFile(hostFileName, false); + for (unsigned int i = 0U; i < hostFile.getCount(); i++) { + std::string reflector = hostFile.getName(i); + in_addr address = CUDPReaderWriter::lookup(hostFile.getAddress(i)); + bool lock = hostFile.getLock(i); + + if (address.s_addr != INADDR_NONE) { + unsigned char* ucp = (unsigned char*)&address; + + std::string addrText; + addrText = CStringUtils::string_format("%u.%u.%u.%u", ucp[0U] & 0xFFU, ucp[1U] & 0xFFU, ucp[2U] & 0xFFU, ucp[3U] & 0xFFU); + + if (lock) + CLog::logInfo("Locking %s to %s", reflector.c_str(), addrText.c_str()); + + reflector.resize(LONG_CALLSIGN_LENGTH - 1U, ' '); + reflector += "G"; + m_cache->updateGateway(reflector, addrText, proto, lock, true); + + count++; + } + } + + std::string protoString; + switch (proto) + { + case DP_DEXTRA: + protoString = "DExtra"; + break; + case DP_DCS: + protoString = "DCS"; + break; + case DP_DPLUS: + protoString = "DPlus"; + break; + default: + // ??? + break; + } + + CLog::logInfo("Loaded %u of %u %s hosts from %s", count, hostFile.getCount(), protoString.c_str() , hostFileName.c_str()); +} \ No newline at end of file diff --git a/DStarGateway/HostsFilesManager.h b/DStarGateway/HostsFilesManager.h new file mode 100644 index 0000000..6e624f4 --- /dev/null +++ b/DStarGateway/HostsFilesManager.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024 by Geoffrey F4FXL / KC3FRA + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#ifndef HostsFilesManager_H +#define HostsFilesManager_H + +#include +#include + +#include "CacheManager.h" +#include "Timer.h" +#include "DStarDefines.h" + +class CHostsFilesManager { +public: + static void setHostFilesDirectories(const std::string & hostFilesDir, const std::string & customHostFilesDir); + static void setDextra(bool enabled, const std::string & dextraUrl); + static void setDCS(bool enabled, const std::string & dcsUrl); + static void setDPlus(bool enabled, const std::string & dplusUrl); + static void setXLX(bool enabled, const std::string & xlxUrl); + static void setCache(CCacheManager * cache); + static void clock(unsigned int ms); + static void setDownloadTimeout(unsigned int seconds); + static bool UpdateHostsFromInternet(); + static bool UpdateHostsFromLocal(); + static bool UpdateHosts(); + static std::future UpdateHostsAsync(); + +private: + static std::string m_hostFilesDirectory; + static std::string m_customFilesDirectory; + + static bool m_dextraEnabled; + static bool m_dcsEnabled; + static bool m_dplusEnabled; + static bool m_xlxEnabled; + + static std::string m_dextraUrl; + static std::string m_dcsUrl; + static std::string m_dplusUrl; + static std::string m_xlxUrl; + + static CCacheManager * m_cache; + static CTimer m_downloadTimer; + + static void loadReflectors(const std::string & directory); + static void loadReflectors(const std::string & hostFileName, DSTAR_PROTOCOL proto); +}; + +#endif \ No newline at end of file From 5fb1530771ed5680f9db6f3295bee181848ea441 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Mon, 16 Sep 2024 20:07:49 +0200 Subject: [PATCH 08/23] typo #50 --- BaseCommon/Daemon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BaseCommon/Daemon.cpp b/BaseCommon/Daemon.cpp index 4c4b193..8baf525 100644 --- a/BaseCommon/Daemon.cpp +++ b/BaseCommon/Daemon.cpp @@ -113,7 +113,7 @@ DAEMONIZE_RESULT CDaemon::daemonise(const std::string& pidFile, const std::strin #ifdef DOUBLE_FORK // Fork off for the second time. Some litterature says it is best to fork 2 times so that the process never can open a terminal. - // However it messes up systemd, event when unit is set as forking + // However it messes up systemd, even when unit is set as forking pid = fork(); // An error occurred From 68c1562bf90b774c96847cbb9253d5c9a2982f02 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Mon, 16 Sep 2024 20:25:31 +0200 Subject: [PATCH 09/23] Update hosts files on signal USR1 #50 --- DStarGateway/DStarGatewayApp.cpp | 16 +++++++++++++--- DStarGateway/DStarGatewayApp.h | 3 ++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/DStarGateway/DStarGatewayApp.cpp b/DStarGateway/DStarGatewayApp.cpp index 7ee437e..2df277b 100644 --- a/DStarGateway/DStarGatewayApp.cpp +++ b/DStarGateway/DStarGatewayApp.cpp @@ -68,8 +68,9 @@ int main(int argc, char *argv[]) signal(SIGILL, CDStarGatewayApp::sigHandlerFatal); signal(SIGFPE, CDStarGatewayApp::sigHandlerFatal); signal(SIGABRT, CDStarGatewayApp::sigHandlerFatal); - signal(SIGTERM, CDStarGatewayApp::sigHandler); - signal(SIGINT, CDStarGatewayApp::sigHandler); + signal(SIGTERM, CDStarGatewayApp::sigHandlerExit); + signal(SIGINT, CDStarGatewayApp::sigHandlerExit); + signal(SIGUSR1, CDStarGatewayApp::sigHandlerUSR); setbuf(stdout, NULL); if (2 != argc) { @@ -370,7 +371,7 @@ bool CDStarGatewayApp::createThread() return true; } -void CDStarGatewayApp::sigHandler(int sig) +void CDStarGatewayApp::sigHandlerExit(int sig) { CLog::logInfo("Caught signal : %s, shutting down gateway", strsignal(sig)); @@ -392,6 +393,15 @@ void CDStarGatewayApp::sigHandlerFatal(int sig) exit(3); } +void CDStarGatewayApp::sigHandlerUSR(int sig) +{ + if(sig == SIGUSR1) { + CLog::logInfo("Caught signal : %s, updating host files", strsignal(sig)); + + CHostsFilesManager::UpdateHostsAsync(); // call and forget + } +} + void CDStarGatewayApp::terminateHandler() { #ifdef DEBUG_DSTARGW diff --git a/DStarGateway/DStarGatewayApp.h b/DStarGateway/DStarGatewayApp.h index 6e5aa21..020a519 100644 --- a/DStarGateway/DStarGatewayApp.h +++ b/DStarGateway/DStarGatewayApp.h @@ -38,6 +38,7 @@ public: void run(); static void sigHandlerFatal(int sig); - static void sigHandler(int sig); + static void sigHandlerExit(int sig); + static void sigHandlerUSR(int sig); static void terminateHandler(); }; From 340c322bf98214cf5d207b98e77986ae37b3ebde Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 20:37:46 +0200 Subject: [PATCH 10/23] add own config for Hostsfiles stuff #50 --- BaseCommon/Config.cpp | 2 +- BaseCommon/Config.h | 2 +- DStarGateway/DStarGatewayApp.cpp | 5 ++++- DStarGateway/DStarGatewayConfig.cpp | 26 +++++++++++++++++++++++--- DStarGateway/DStarGatewayConfig.h | 10 +++++++++- DStarGateway/example.cfg | 10 +++++++--- 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/BaseCommon/Config.cpp b/BaseCommon/Config.cpp index ffe8468..d8a5cba 100644 --- a/BaseCommon/Config.cpp +++ b/BaseCommon/Config.cpp @@ -170,7 +170,7 @@ bool CConfig::getValue(const std::string §ion, const std::string& key, doubl return true; } -bool CConfig::getValue(const std::string §ion, const std::string& key, unsigned int &value, unsigned int min, unsigned int max, int defaultValue) const +bool CConfig::getValue(const std::string §ion, const std::string& key, unsigned int &value, unsigned int min, unsigned int max, unsigned int defaultValue) const { TConfigValue * val = lookupValue(section, key); if(val == nullptr || val->m_value.empty()) { diff --git a/BaseCommon/Config.h b/BaseCommon/Config.h index aabf432..e0a523e 100644 --- a/BaseCommon/Config.h +++ b/BaseCommon/Config.h @@ -99,7 +99,7 @@ public: bool getValue(const std::string §ion, const std::string& key, bool &value, bool defaultValue) const; bool getValue(const std::string §ion, const std::string& key, int &value, int min, int max, int defaultValue) const; bool getValue(const std::string §ion, const std::string& key, double &value, double min, double max, double defaultValue) const; - bool getValue(const std::string §ion, const std::string& key, unsigned int &value, unsigned int min, unsigned int max,int defaultValue) const; + bool getValue(const std::string §ion, const std::string& key, unsigned int &value, unsigned int min, unsigned int max, unsigned int defaultValue) const; bool getValue(const std::string §ion, const std::string& key, unsigned char &value, unsigned char min, unsigned char max,unsigned char defaultValue) const; bool getValue(const std::string §ion, const std::string& key, std::string &value, unsigned int minLen, unsigned int maxLen, const std::string defaultValue) const; bool getValue(const std::string §ion, const std::string& key, std::string &value, const std::string defaultValue, const std::vector& allowedValues) const; diff --git a/DStarGateway/DStarGatewayApp.cpp b/DStarGateway/DStarGatewayApp.cpp index 2df277b..5b3301c 100644 --- a/DStarGateway/DStarGatewayApp.cpp +++ b/DStarGateway/DStarGatewayApp.cpp @@ -347,7 +347,10 @@ bool CDStarGatewayApp::createThread() m_thread->setXLX(xlxConfig.enabled); // Setup hostsfiles - CHostsFilesManager::setHostFilesDirectories(paths.dataDir, paths.customHostsFiles); + THostsFiles hostsFilesConfig; + m_config->getHostsFiles(hostsFilesConfig); + CHostsFilesManager::setHostFilesDirectories(hostsFilesConfig.downloadedHostFiles, hostsFilesConfig.customHostsFiles); + CHostsFilesManager::setDownloadTimeout(3600 * hostsFilesConfig.downloadTimeout); CHostsFilesManager::setDextra(dextraConfig.enabled, dextraConfig.hostfileUrl); CHostsFilesManager::setDCS (dcsConfig.enabled, dcsConfig.hostfileUrl); CHostsFilesManager::setDPlus (dplusConfig.enabled, dplusConfig.hostfileUrl); diff --git a/DStarGateway/DStarGatewayConfig.cpp b/DStarGateway/DStarGatewayConfig.cpp index 2438e87..0833678 100644 --- a/DStarGateway/DStarGatewayConfig.cpp +++ b/DStarGateway/DStarGatewayConfig.cpp @@ -45,6 +45,7 @@ bool CDStarGatewayConfig::load() ret = loadIrcDDB(cfg) && ret; ret = loadRepeaters(cfg) && ret; ret = loadPaths(cfg) && ret; + ret = loadHostsFiles(cfg) && ret; ret = loadLog(cfg) && ret; ret = loadAPRS(cfg) && ret; ret = loadDextra(cfg) && ret; @@ -195,14 +196,28 @@ bool CDStarGatewayConfig::loadLog(const CConfig & cfg) bool CDStarGatewayConfig::loadPaths(const CConfig & cfg) { bool ret = cfg.getValue("paths", "data", m_paths.dataDir, 0, 2048, "/usr/local/share/dstargateway.d/"); - ret = cfg.getValue("customHostfiles", "data", m_paths.customHostsFiles, 0, 2048, "/usr/local/share/dstargateway.d/hostfiles.d/") && ret; if(ret && m_paths.dataDir[m_paths.dataDir.length() - 1] != '/') { m_paths.dataDir.push_back('/'); } - if(ret && m_paths.dataDir[m_paths.customHostsFiles.length() - 1] != '/') { - m_paths.customHostsFiles.push_back('/'); + //TODO 20211226 check if directory are accessible + + return ret; +} + +bool CDStarGatewayConfig::loadHostsFiles(const CConfig & cfg) +{ + bool ret = cfg.getValue("HostsFiles", "downloadedHostsFiles", m_hostsFiles.downloadedHostFiles, 0, 2048, "/usr/local/share/dstargateway.d/"); + ret = cfg.getValue("HostsFiles", "customHostsfiles", m_hostsFiles.customHostsFiles, 0, 2048, "/usr/local/share/dstargateway.d/hostsfiles.d/"); + ret = cfg.getValue("HostsFiles", "downloadTimer", m_hostsFiles.downloadTimeout, 24U, 0xffffffffU, 72U); + + if(ret && m_hostsFiles.downloadedHostFiles[m_hostsFiles.downloadedHostFiles.length() - 1] != '/') { + m_hostsFiles.downloadedHostFiles.push_back('/'); + } + + if(ret && m_hostsFiles.customHostsFiles[m_hostsFiles.customHostsFiles.length() - 1] != '/') { + m_hostsFiles.downloadedHostFiles.push_back('/'); } //TODO 20211226 check if directory are accessible @@ -441,6 +456,11 @@ void CDStarGatewayConfig::getPaths(Tpaths & paths) const paths = m_paths; } +void CDStarGatewayConfig::getHostsFiles(THostsFiles & hostsFiles) const +{ + hostsFiles = m_hostsFiles; +} + void CDStarGatewayConfig::getAPRS(TAPRS & aprs) const { aprs = m_aprs; diff --git a/DStarGateway/DStarGatewayConfig.h b/DStarGateway/DStarGatewayConfig.h index 6a620e5..2d5bf62 100644 --- a/DStarGateway/DStarGatewayConfig.h +++ b/DStarGateway/DStarGatewayConfig.h @@ -80,9 +80,14 @@ typedef struct { typedef struct { std::string dataDir; - std::string customHostsFiles; } Tpaths; +typedef struct { + std::string downloadedHostFiles; + std::string customHostsFiles; + unsigned int downloadTimeout; +} THostsFiles; + typedef struct { std::string logDir; LOG_SEVERITY displayLevel; @@ -160,6 +165,7 @@ public: unsigned int getRepeaterCount() const; void getLog(TLog& log) const; void getPaths(Tpaths & paths) const; + void getHostsFiles(THostsFiles & hostsFiles) const; void getAPRS(TAPRS & aprs) const; void getDExtra(TDextra & dextra) const; void getDPlus(TDplus & dplus) const; @@ -180,6 +186,7 @@ private: bool loadRepeaters(const CConfig & cfg); bool loadLog(const CConfig & cfg); bool loadPaths(const CConfig & cfg); + bool loadHostsFiles(const CConfig & cfg); bool loadAPRS(const CConfig & cfg); bool loadDextra(const CConfig & cfg); bool loadDPlus(const CConfig & cfg); @@ -196,6 +203,7 @@ private: std::string m_fileName; TGateway m_gateway; Tpaths m_paths; + THostsFiles m_hostsFiles; TAPRS m_aprs; TDextra m_dextra; TDplus m_dplus; diff --git a/DStarGateway/example.cfg b/DStarGateway/example.cfg index 23ae692..e385857 100644 --- a/DStarGateway/example.cfg +++ b/DStarGateway/example.cfg @@ -155,10 +155,14 @@ displayLevel= # defaults to info, valid values are trace, debug, info, warning repeatThreshold=#defaults to 2, valid values are disbaled and 1 to 10. Prevents flooding of logs from repeated log messages. [Paths] -data=/usr/local/share/dstargateway.d/ # Path where the data (auto downbloaded hostfiles, audio files etc) can be found -customHostfiles=/usr/local/share/dstargateway.d/hostfiles.d/ # Place your custom host files in this directory, this dir must be different from datadir. - # Any hosts found here will override same host in downloaded files +data=/usr/local/share/dstargateway.d/ # Path where the data (audio files etc) can be found +[HostsFiles] +downloadedHostsFiles=/usr/local/share/dstargateway.d/ # Path where the downloaded host files are stored, make sure user dstar has write access +downloadTimer= # Redownload host files every X hours, must be greater than 24 hours. Defaults to 72 hours +customHostsfiles=/usr/local/share/dstargateway.d/hostfiles.d/ # Place your custom host files in this directory, this dir must be different from datadir. + # Any hosts found here will override same host in downloaded files + [DExtra] enabled=true # There is no reason to disable this maxDongles=5 From b8cca42b8563c4b884c63cd59a435fb18c9f676f Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 20:39:34 +0200 Subject: [PATCH 11/23] make sure DATA_DIR is writable by dstar user #50 --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index d7daf6d..a9e1485 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,7 @@ install : DStarGateway/dstargateway DGWRemoteControl/dgwremotecontrol @mkdir -p $(BIN_DIR) @mkdir -p $(LOG_DIR) @chown dstar:dstar $(LOG_DIR) + @chown dstar:dstar $(DATA_DIR) # Install data $(MAKE) -C Data install From 04f9d83de4b3c0643f1dbcedd192a6e8c3fd4269 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 20:43:58 +0200 Subject: [PATCH 12/23] make sure downlaod timer is started --- DStarGateway/HostsFilesManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DStarGateway/HostsFilesManager.cpp b/DStarGateway/HostsFilesManager.cpp index 60805d0..894266d 100644 --- a/DStarGateway/HostsFilesManager.cpp +++ b/DStarGateway/HostsFilesManager.cpp @@ -83,12 +83,13 @@ void CHostsFilesManager::clock(unsigned int ms) if(m_downloadTimer.hasExpired()) { UpdateHostsAsync(); // call and forget + m_downloadTimer.start(); } } void CHostsFilesManager::setDownloadTimeout(unsigned int seconds) { - m_downloadTimer.setTimeout(seconds); + m_downloadTimer.start(seconds); } bool CHostsFilesManager::UpdateHostsFromInternet() From 4163f4df113001beb7a4c6cc0cb5848346d5f2ed Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 20:48:50 +0200 Subject: [PATCH 13/23] Add some logging #50 --- DStarGateway/HostsFilesManager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/DStarGateway/HostsFilesManager.cpp b/DStarGateway/HostsFilesManager.cpp index 894266d..9158e03 100644 --- a/DStarGateway/HostsFilesManager.cpp +++ b/DStarGateway/HostsFilesManager.cpp @@ -82,6 +82,7 @@ void CHostsFilesManager::clock(unsigned int ms) m_downloadTimer.clock(ms); if(m_downloadTimer.hasExpired()) { + CLog::logInfo("Downloading hosts files after %u hours", m_downloadTimer.getTimeout() / 3600U); UpdateHostsAsync(); // call and forget m_downloadTimer.start(); } From 3c0bf94b588474a580efc03c522073b54a7fb26e Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 20:58:06 +0200 Subject: [PATCH 14/23] vscode crap --- .vscode/settings.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index df80dfe..476ada1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -81,7 +81,13 @@ "numeric": "cpp", "regex": "cpp", "cfenv": "cpp", - "cinttypes": "cpp" + "cinttypes": "cpp", + "charconv": "cpp", + "source_location": "cpp", + "format": "cpp", + "shared_mutex": "cpp", + "span": "cpp", + "text_encoding": "cpp" }, "files.exclude": { "**/.DS_Store": true, From ec9518fc4d5ac1079f3b7958ad0f12460235e52b Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 20:58:39 +0200 Subject: [PATCH 15/23] Handle unable to copy downloaded file #50 --- Common/HostsFileDownloader.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Common/HostsFileDownloader.cpp b/Common/HostsFileDownloader.cpp index 74fdb29..765f6b0 100644 --- a/Common/HostsFileDownloader.cpp +++ b/Common/HostsFileDownloader.cpp @@ -59,10 +59,17 @@ bool CHostsFileDownloader::download(const std::string & hostsFileURL, const std: if(ok) { outFileName = std::string(outFileNameBuf); - ok = ok && std::filesystem::copy_file(outFileName, hostFilePath, std::filesystem::copy_options::overwrite_existing); - if(!ok) { - CLog::logError("Failed to copy host file to %s", hostFilePath.c_str()); + + try + { + std::filesystem::copy_file(outFileName, hostFilePath, std::filesystem::copy_options::overwrite_existing); + } + catch(std::filesystem::filesystem_error& e) + { + ok = false; + CLog::logError("Failed to copy host file to %s: %s", hostFilePath.c_str(), e.what()); } + } else { CLog::logWarning("Failed to download Host file from %s, using previous file", hostsFileURL.c_str()); } From 1532945982c2374f529a9480129b86a706244c24 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 21:49:51 +0200 Subject: [PATCH 16/23] correct typo in xlx file name #50 --- DStarGateway/HostsFilesManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DStarGateway/HostsFilesManager.cpp b/DStarGateway/HostsFilesManager.cpp index 9158e03..ae6134d 100644 --- a/DStarGateway/HostsFilesManager.cpp +++ b/DStarGateway/HostsFilesManager.cpp @@ -132,7 +132,7 @@ std::future CHostsFilesManager::UpdateHostsAsync() void CHostsFilesManager::loadReflectors(const std::string & directory) { if (m_xlxEnabled) { - std::string fileName = directory + "/" + DPLUS_HOSTS_FILE_NAME; + std::string fileName = directory + "/" + DCS_HOSTS_FILE_NAME; loadReflectors(fileName, DP_DCS); } From 33b7c4c6755d998780e4f9c533594e179c59ee3b Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 22:00:50 +0200 Subject: [PATCH 17/23] again correct type in file name, time to go to bed #50 --- DStarGateway/HostsFilesManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DStarGateway/HostsFilesManager.cpp b/DStarGateway/HostsFilesManager.cpp index ae6134d..be06017 100644 --- a/DStarGateway/HostsFilesManager.cpp +++ b/DStarGateway/HostsFilesManager.cpp @@ -132,7 +132,7 @@ std::future CHostsFilesManager::UpdateHostsAsync() void CHostsFilesManager::loadReflectors(const std::string & directory) { if (m_xlxEnabled) { - std::string fileName = directory + "/" + DCS_HOSTS_FILE_NAME; + std::string fileName = directory + "/" + XLX_HOSTS_FILE_NAME; loadReflectors(fileName, DP_DCS); } From ebbbd595b53e6132dfb38654d234fa4868f5ecf8 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Tue, 17 Sep 2024 22:06:10 +0200 Subject: [PATCH 18/23] do not load hosts asynchronously on startup, it is confusing #50 --- DStarGateway/DStarGatewayThread.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DStarGateway/DStarGatewayThread.cpp b/DStarGateway/DStarGatewayThread.cpp index 0a65d90..c2d1ff1 100644 --- a/DStarGateway/DStarGatewayThread.cpp +++ b/DStarGateway/DStarGatewayThread.cpp @@ -138,7 +138,7 @@ CDStarGatewayThread::~CDStarGatewayThread() void* CDStarGatewayThread::Entry() { CHostsFilesManager::setCache(&m_cache); - auto hostFut = CHostsFilesManager::UpdateHostsAsync(); // Do this in a separate thread + CHostsFilesManager::UpdateHosts(); // Truncate the old Links.log file std::string fullName = m_logDir + "/" + LINKS_BASE_NAME + ".log"; @@ -199,7 +199,6 @@ void* CDStarGatewayThread::Entry() } // Wait here until we have the essentials to run - hostFut.get(); while (!m_killed && (m_dextraPool == NULL || m_dplusPool == NULL || m_dcsPool == NULL || m_g2HandlerPool == NULL || (m_icomRepeaterHandler == NULL && m_hbRepeaterHandler == NULL && m_dummyRepeaterHandler == NULL) || m_gatewayCallsign.empty())) ::std::this_thread::sleep_for(std::chrono::milliseconds(500UL)); // 1/2 sec From 2f2adedbc5a4060395f75d5ffa9654eb8970055c Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Sun, 22 Sep 2024 07:37:46 +0200 Subject: [PATCH 19/23] Move HostsFilesManager to Common #50 --- {DStarGateway => Common}/HostsFilesManager.cpp | 0 {DStarGateway => Common}/HostsFilesManager.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {DStarGateway => Common}/HostsFilesManager.cpp (100%) rename {DStarGateway => Common}/HostsFilesManager.h (100%) diff --git a/DStarGateway/HostsFilesManager.cpp b/Common/HostsFilesManager.cpp similarity index 100% rename from DStarGateway/HostsFilesManager.cpp rename to Common/HostsFilesManager.cpp diff --git a/DStarGateway/HostsFilesManager.h b/Common/HostsFilesManager.h similarity index 100% rename from DStarGateway/HostsFilesManager.h rename to Common/HostsFilesManager.h From eb5525b6fde7a60ba11113c55dd9468790c02f9d Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Sun, 22 Sep 2024 08:37:41 +0200 Subject: [PATCH 20/23] losen dependency #50 --- Common/HostsFilesManager.cpp | 16 +++++++++++----- Common/HostsFilesManager.h | 5 +++++ DStarGateway/DStarGatewayThread.cpp | 2 ++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Common/HostsFilesManager.cpp b/Common/HostsFilesManager.cpp index be06017..34c689f 100644 --- a/Common/HostsFilesManager.cpp +++ b/Common/HostsFilesManager.cpp @@ -19,7 +19,6 @@ #include #include "HostsFilesManager.h" -#include "HostsFileDownloader.h" #include "HostFile.h" #include "StringUtils.h" #include "UDPReaderWriter.h" @@ -41,12 +40,19 @@ std::string CHostsFilesManager::m_xlxUrl(""); CCacheManager * CHostsFilesManager::m_cache = nullptr; CTimer CHostsFilesManager::m_downloadTimer(1000U, 60 * 60 * 24); +HostFileDownloadCallback CHostsFilesManager::m_downloadCallback = nullptr; + void CHostsFilesManager::setHostFilesDirectories(const std::string & hostFilesDir, const std::string & customHostFilesDir) { m_hostFilesDirectory.assign(hostFilesDir); m_customFilesDirectory.assign(customHostFilesDir); } +void CHostsFilesManager::setDownloadCallback(HostFileDownloadCallback callback) +{ + m_downloadCallback = callback; +} + void CHostsFilesManager::setDextra(bool enabled, const std::string & url) { m_dextraEnabled = enabled; @@ -97,10 +103,10 @@ bool CHostsFilesManager::UpdateHostsFromInternet() { CLog::logInfo("Updating hosts files from internet"); bool ret = true; - if(m_dextraEnabled && !m_dextraUrl.empty()) ret = CHostsFileDownloader::download(m_dextraUrl, m_hostFilesDirectory + "/" + DEXTRA_HOSTS_FILE_NAME) && ret; - if(m_dcsEnabled && !m_dcsUrl.empty()) ret = CHostsFileDownloader::download(m_dcsUrl, m_hostFilesDirectory + "/" + DCS_HOSTS_FILE_NAME) && ret; - if(m_dplusEnabled && !m_dplusUrl.empty()) ret = CHostsFileDownloader::download(m_dplusUrl, m_hostFilesDirectory + "/" + DPLUS_HOSTS_FILE_NAME) && ret; - if(m_xlxEnabled && !m_xlxUrl.empty()) ret = CHostsFileDownloader::download(m_xlxUrl, m_hostFilesDirectory + "/" + XLX_HOSTS_FILE_NAME) && ret; + if(m_dextraEnabled && !m_dextraUrl.empty()) ret = m_downloadCallback(m_dextraUrl, m_hostFilesDirectory + "/" + DEXTRA_HOSTS_FILE_NAME) && ret; + if(m_dcsEnabled && !m_dcsUrl.empty()) ret = m_downloadCallback(m_dcsUrl, m_hostFilesDirectory + "/" + DCS_HOSTS_FILE_NAME) && ret; + if(m_dplusEnabled && !m_dplusUrl.empty()) ret = m_downloadCallback(m_dplusUrl, m_hostFilesDirectory + "/" + DPLUS_HOSTS_FILE_NAME) && ret; + if(m_xlxEnabled && !m_xlxUrl.empty()) ret = m_downloadCallback(m_xlxUrl, m_hostFilesDirectory + "/" + XLX_HOSTS_FILE_NAME) && ret; if(!ret) CLog::logWarning("Some hosts files failed to downlaod"); diff --git a/Common/HostsFilesManager.h b/Common/HostsFilesManager.h index 6e624f4..9775cd7 100644 --- a/Common/HostsFilesManager.h +++ b/Common/HostsFilesManager.h @@ -26,9 +26,12 @@ #include "Timer.h" #include "DStarDefines.h" +typedef bool (*HostFileDownloadCallback)(const std::string &, const std::string &); + class CHostsFilesManager { public: static void setHostFilesDirectories(const std::string & hostFilesDir, const std::string & customHostFilesDir); + static void setDownloadCallback(HostFileDownloadCallback downloadCallback); static void setDextra(bool enabled, const std::string & dextraUrl); static void setDCS(bool enabled, const std::string & dcsUrl); static void setDPlus(bool enabled, const std::string & dplusUrl); @@ -58,6 +61,8 @@ private: static CCacheManager * m_cache; static CTimer m_downloadTimer; + static HostFileDownloadCallback m_downloadCallback; + static void loadReflectors(const std::string & directory); static void loadReflectors(const std::string & hostFileName, DSTAR_PROTOCOL proto); }; diff --git a/DStarGateway/DStarGatewayThread.cpp b/DStarGateway/DStarGatewayThread.cpp index c2d1ff1..a6f204d 100644 --- a/DStarGateway/DStarGatewayThread.cpp +++ b/DStarGateway/DStarGatewayThread.cpp @@ -55,6 +55,7 @@ #include "Log.h" #include "StringUtils.h" #include "HostsFilesManager.h" +#include "HostsFileDownloader.h" const std::string LOOPBACK_ADDRESS("127.0.0.1"); @@ -138,6 +139,7 @@ CDStarGatewayThread::~CDStarGatewayThread() void* CDStarGatewayThread::Entry() { CHostsFilesManager::setCache(&m_cache); + CHostsFilesManager::setDownloadCallback(CHostsFileDownloader::download); CHostsFilesManager::UpdateHosts(); // Truncate the old Links.log file From 61e4d9f8103e9d5f2dc5c9298968116012d95520 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Mon, 23 Sep 2024 06:16:38 +0200 Subject: [PATCH 21/23] Add tests for hosts files manager #50 --- .vscode/tasks.json | 10 +- Tests/HostsFilesManager/UpdateHosts.cpp | 207 ++++++++++++++++++ Tests/HostsFilesManager/custom/DCS_Hosts.txt | 1 + .../HostsFilesManager/custom/DExtra_Hosts.txt | 1 + .../HostsFilesManager/custom/DPlus_Hosts.txt | 1 + Tests/HostsFilesManager/custom/XLX_Hosts.txt | 1 + .../HostsFilesManager/internet/DCS_Hosts.txt | 1 + .../internet/DExtra_Hosts.txt | 1 + .../internet/DPlus_Hosts.txt | 1 + .../HostsFilesManager/internet/XLX_Hosts.txt | 1 + 10 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 Tests/HostsFilesManager/UpdateHosts.cpp create mode 100644 Tests/HostsFilesManager/custom/DCS_Hosts.txt create mode 100644 Tests/HostsFilesManager/custom/DExtra_Hosts.txt create mode 100644 Tests/HostsFilesManager/custom/DPlus_Hosts.txt create mode 100644 Tests/HostsFilesManager/custom/XLX_Hosts.txt create mode 100644 Tests/HostsFilesManager/internet/DCS_Hosts.txt create mode 100644 Tests/HostsFilesManager/internet/DExtra_Hosts.txt create mode 100644 Tests/HostsFilesManager/internet/DPlus_Hosts.txt create mode 100644 Tests/HostsFilesManager/internet/XLX_Hosts.txt diff --git a/.vscode/tasks.json b/.vscode/tasks.json index bf6c999..9f00615 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -26,10 +26,7 @@ "USE_GPSD=1", "DStarGateway/dstargateway" ], - "group": { - "kind": "build", - "isDefault": true - }, + "group": "build", "problemMatcher": [] }, { @@ -94,7 +91,10 @@ "ENABLE_DEBUG=1", "USE_GPSD=1" ], - "group": "build", + "group": { + "kind": "build", + "isDefault": true + }, "problemMatcher": [] } ] diff --git a/Tests/HostsFilesManager/UpdateHosts.cpp b/Tests/HostsFilesManager/UpdateHosts.cpp new file mode 100644 index 0000000..91d399f --- /dev/null +++ b/Tests/HostsFilesManager/UpdateHosts.cpp @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2021-2024 by Geoffrey Merck F4FXL / KC3FRA + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include + +#include "HostsFilesManager.h" +#include "CacheManager.h" +#include "DStarDefines.h" + +namespace HostsFilesManagerTests +{ + class HostsFilesManager_UpdateHosts : public ::testing::Test + { + protected: + std::string m_internetPath; + std::string m_customPath; + CCacheManager * m_cache; + + HostsFilesManager_UpdateHosts() : + m_internetPath(), + m_customPath(), + m_cache(nullptr) + { + + } + + + void SetUp() override + { + char buf[2048]; + auto size = ::readlink("/proc/self/exe", buf, 2048); + if(size > 0) { + std::string path(buf, size); + auto lastSlashPos = path.find_last_of('/'); + path.resize(lastSlashPos); + + m_internetPath = path + "/HostsFilesManager/internet/"; + m_customPath = path + "/HostsFilesManager/custom/"; + } + + CHostsFilesManager::setDCS(false, ""); + CHostsFilesManager::setDextra(false, ""); + CHostsFilesManager::setXLX(false, ""); + CHostsFilesManager::setDPlus(false, ""); + + if(m_cache != nullptr) delete m_cache; + m_cache = new CCacheManager(); + } + + static bool dummyDownload(const std::string & a, const std::string & b) + { + std::cout << a << std::endl << b; + return true; + } + }; + + TEST_F(HostsFilesManager_UpdateHosts, DExtraFromInternet) + { + CHostsFilesManager::setDextra(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet"); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("XRF123 G"); + + EXPECT_NE(gw, nullptr) << "DExtra host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "XRF123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DEXTRA) << "Protocol mismatch"; + } + + + TEST_F(HostsFilesManager_UpdateHosts, DExtraFromInternetCustomOverride) + { + CHostsFilesManager::setDextra(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("XRF123 G"); + + EXPECT_NE(gw, nullptr) << "DExtra host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "XRF123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DEXTRA) << "Protocol mismatch"; + } + + TEST_F(HostsFilesManager_UpdateHosts, DCSFromInternet) + { + CHostsFilesManager::setDCS(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet"); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("DCS123 G"); + + EXPECT_NE(gw, nullptr) << "DCS host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "DCS123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch"; + } + + + TEST_F(HostsFilesManager_UpdateHosts, DCSFromInternetCustomOverride) + { + CHostsFilesManager::setDCS(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("DCS123 G"); + + EXPECT_NE(gw, nullptr) << "DCS host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "DCS123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch"; + } + + TEST_F(HostsFilesManager_UpdateHosts, DPlusFromInternet) + { + CHostsFilesManager::setDPlus(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet"); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("REF123 G"); + + EXPECT_NE(gw, nullptr) << "DPlus host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "REF123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DPLUS) << "Protocol mismatch"; + } + + + TEST_F(HostsFilesManager_UpdateHosts, DPlusFromInternetCustomOverride) + { + CHostsFilesManager::setDPlus(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("REF123 G"); + + EXPECT_NE(gw, nullptr) << "DPlus host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "REF123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DPLUS) << "Protocol mismatch"; + } + + TEST_F(HostsFilesManager_UpdateHosts, XLXFromInternet) + { + CHostsFilesManager::setXLX(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet"); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("XLX123 G"); + + EXPECT_NE(gw, nullptr) << "XLX host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "XLX123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch"; + } + + + TEST_F(HostsFilesManager_UpdateHosts, XLXFromInternetCustomOverride) + { + CHostsFilesManager::setXLX(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra"); + CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath); + CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload); + CHostsFilesManager::setCache(m_cache); + + CHostsFilesManager::UpdateHosts(); + auto gw = m_cache->findGateway("XLX123 G"); + + EXPECT_NE(gw, nullptr) << "XLX host not found"; + EXPECT_STREQ(gw->getGateway().c_str(), "XLX123 G"); + EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch"; + EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch"; + } +} \ No newline at end of file diff --git a/Tests/HostsFilesManager/custom/DCS_Hosts.txt b/Tests/HostsFilesManager/custom/DCS_Hosts.txt new file mode 100644 index 0000000..5612ca5 --- /dev/null +++ b/Tests/HostsFilesManager/custom/DCS_Hosts.txt @@ -0,0 +1 @@ +DCS123 2.2.2.2 \ No newline at end of file diff --git a/Tests/HostsFilesManager/custom/DExtra_Hosts.txt b/Tests/HostsFilesManager/custom/DExtra_Hosts.txt new file mode 100644 index 0000000..9e68e88 --- /dev/null +++ b/Tests/HostsFilesManager/custom/DExtra_Hosts.txt @@ -0,0 +1 @@ +XRF123 2.2.2.2 \ No newline at end of file diff --git a/Tests/HostsFilesManager/custom/DPlus_Hosts.txt b/Tests/HostsFilesManager/custom/DPlus_Hosts.txt new file mode 100644 index 0000000..b8a56a7 --- /dev/null +++ b/Tests/HostsFilesManager/custom/DPlus_Hosts.txt @@ -0,0 +1 @@ +REF123 2.2.2.2 \ No newline at end of file diff --git a/Tests/HostsFilesManager/custom/XLX_Hosts.txt b/Tests/HostsFilesManager/custom/XLX_Hosts.txt new file mode 100644 index 0000000..d65056c --- /dev/null +++ b/Tests/HostsFilesManager/custom/XLX_Hosts.txt @@ -0,0 +1 @@ +XLX123 2.2.2.2 \ No newline at end of file diff --git a/Tests/HostsFilesManager/internet/DCS_Hosts.txt b/Tests/HostsFilesManager/internet/DCS_Hosts.txt new file mode 100644 index 0000000..3bae50b --- /dev/null +++ b/Tests/HostsFilesManager/internet/DCS_Hosts.txt @@ -0,0 +1 @@ +DCS123 1.1.1.1 \ No newline at end of file diff --git a/Tests/HostsFilesManager/internet/DExtra_Hosts.txt b/Tests/HostsFilesManager/internet/DExtra_Hosts.txt new file mode 100644 index 0000000..7ec1fbf --- /dev/null +++ b/Tests/HostsFilesManager/internet/DExtra_Hosts.txt @@ -0,0 +1 @@ +XRF123 1.1.1.1 \ No newline at end of file diff --git a/Tests/HostsFilesManager/internet/DPlus_Hosts.txt b/Tests/HostsFilesManager/internet/DPlus_Hosts.txt new file mode 100644 index 0000000..3a1a484 --- /dev/null +++ b/Tests/HostsFilesManager/internet/DPlus_Hosts.txt @@ -0,0 +1 @@ +REF123 1.1.1.1 \ No newline at end of file diff --git a/Tests/HostsFilesManager/internet/XLX_Hosts.txt b/Tests/HostsFilesManager/internet/XLX_Hosts.txt new file mode 100644 index 0000000..1d2d8e5 --- /dev/null +++ b/Tests/HostsFilesManager/internet/XLX_Hosts.txt @@ -0,0 +1 @@ +XLX123 1.1.1.1 \ No newline at end of file From af11c3ec4ec354a8e1c0f88c475818b1bf641850 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Mon, 23 Sep 2024 06:19:09 +0200 Subject: [PATCH 22/23] udpate readme #50 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1d0b085..9516f1b 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ The testing framwework used is Google Test. # 6. Version History ## 6.1. Version 1.0 +- [**Improvement**] Automatically download hosts files on startup. It is also possible to specify custom hosts files to override hosts from the internet. Needs some configuration adjustement, see example.cfg. ([#50](https://github.com/F4FXL/DStarGateway/issues/50)) - [**Improvement**] Limit log reporting ([#44](https://github.com/F4FXL/DStarGateway/issues/44)) - [**Improvement**] Improve CI to include all variants of build configurations ([#40](https://github.com/F4FXL/DStarGateway/issues/40)) - [**Bugfix**] Fix #43 Cache not updated when answering ircddb gateway is only conected to one network ([#43](https://github.com/F4FXL/DStarGateway/issues/43)) From 43767d4a31478a1f22da50aa65c06a0f295ed582 Mon Sep 17 00:00:00 2001 From: Geoffrey Merck Date: Mon, 23 Sep 2024 06:35:06 +0200 Subject: [PATCH 23/23] Update readme #50 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9516f1b..c1d367d 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ All the features found in ircddbGateway are supposed to be working. Except the o - DPlus, DExtra and G2 NAT Traversal using ircddb network as rendez-vous server. I.e. it is not required to open firewall ports for Callsign Routing or Gateway calls. however it is still recommended to do so. But NAT Traversal will bring more flexibility when operating on CGNAT (Mobile) Networks. - Forward RSMS1A app messages from/to APRS-IS Network, yes you can send/receive messages to and from aprs. Yes, you can send messages to APRS stations and Vice Versa. Additionnally, part of the message is sent as Text Dat in the slow data. This allows you to read the message directly on your radio screen. - Repeater Link status is sent to APRS-IS as a status frame +- Built-in download of hosts files # 3. Building and installing ## 3.1. Initial setup @@ -134,6 +135,7 @@ To update host files, from within the source code directory, run sudo make newhostfiles sudo systemctl restart dstargateway.service ``` +or juste configure and run the software and it will download latest host files. # 4. Dashboard @johnhays K7VE has developed a nice lightweight NodeJS dashboard. Code and instructions can be found on his [GitHub](https://github.com/johnhays/dsgwdashboard).