Merge branch 'feature/auto_host_file_update_#50' into develop closes #50

develop
Geoffrey Merck 1 year ago
commit ea23834643

@ -81,7 +81,13 @@
"numeric": "cpp",
"regex": "cpp",
"cfenv": "cpp",
"cinttypes": "cpp"
"cinttypes": "cpp",
"charconv": "cpp",
"source_location": "cpp",
"format": "cpp",
"shared_mutex": "cpp",
"span": "cpp",
"text_encoding": "cpp"
},
"files.exclude": {
"**/.DS_Store": true,

@ -170,7 +170,7 @@ bool CConfig::getValue(const std::string &section, const std::string& key, doubl
return true;
}
bool CConfig::getValue(const std::string &section, const std::string& key, unsigned int &value, unsigned int min, unsigned int max, int defaultValue) const
bool CConfig::getValue(const std::string &section, const std::string& key, unsigned int &value, unsigned int min, unsigned int max, unsigned int defaultValue) const
{
TConfigValue * val = lookupValue(section, key);
if(val == nullptr || val->m_value.empty()) {

@ -99,7 +99,7 @@ public:
bool getValue(const std::string &section, const std::string& key, bool &value, bool defaultValue) const;
bool getValue(const std::string &section, const std::string& key, int &value, int min, int max, int defaultValue) const;
bool getValue(const std::string &section, const std::string& key, double &value, double min, double max, double defaultValue) const;
bool getValue(const std::string &section, const std::string& key, unsigned int &value, unsigned int min, unsigned int max,int defaultValue) const;
bool getValue(const std::string &section, const std::string& key, unsigned int &value, unsigned int min, unsigned int max, unsigned int defaultValue) const;
bool getValue(const std::string &section, const std::string& key, unsigned char &value, unsigned char min, unsigned char max,unsigned char defaultValue) const;
bool getValue(const std::string &section, const std::string& key, std::string &value, unsigned int minLen, unsigned int maxLen, const std::string defaultValue) const;
bool getValue(const std::string &section, const std::string& key, std::string &value, const std::string defaultValue, const std::vector<std::string>& allowedValues) const;

@ -113,7 +113,7 @@ DAEMONIZE_RESULT CDaemon::daemonise(const std::string& pidFile, const std::strin
#ifdef DOUBLE_FORK
// Fork off for the second time. Some litterature says it is best to fork 2 times so that the process never can open a terminal.
// However it messes up systemd, event when unit is set as forking
// However it messes up systemd, even when unit is set as forking
pid = fork();
// An error occurred

@ -25,6 +25,7 @@
const std::string DEXTRA_HOSTS_FILE_NAME("DExtra_Hosts.txt");
const std::string DCS_HOSTS_FILE_NAME("DCS_Hosts.txt");
const std::string DPLUS_HOSTS_FILE_NAME("DPlus_Hosts.txt");
const std::string XLX_HOSTS_FILE_NAME("XLX_Hosts.txt");
const std::string GATEWAY_HOSTS_FILE_NAME("Gateway_Hosts.txt");
const std::string LINKS_BASE_NAME("Links");

@ -18,33 +18,34 @@
#include <stdio.h>
#include <cstdio>
#include <filesystem>
#include <curl/curl.h>
#include "XLXHostsFileDownloader.h"
#include "HostsFileDownloader.h"
#include "Log.h"
size_t CXLXHostsFileDownloader::write_data(void *ptr, size_t size, size_t nmemb, FILE *stream)
size_t CHostsFileDownloader::write_data(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
/* wxHTTP randomly crashes when called on a worker thread, this must be called from main thread ! */
std::string CXLXHostsFileDownloader::download(const std::string & xlxHostsFileURL)
bool CHostsFileDownloader::download(const std::string & hostsFileURL, const std::string & hostFilePath)
{
CURL *curl;
FILE *fp;
bool ok = false;
std::string outFileName;
char outFileNameBuf[] = "/tmp/XLXHostFile_XXXXXX";
char outFileNameBuf[] = "/tmp/HostFile_XXXXXX";
CLog::logInfo("Downloading XLX host file from %s", xlxHostsFileURL.c_str());
CLog::logInfo("Downloading host file from %s", hostsFileURL.c_str());
curl = curl_easy_init();
if (curl) {
int filedes = mkstemp(outFileNameBuf);
if(filedes > 0 && (fp = fdopen(filedes,"wb")) != NULL) {
curl_easy_setopt(curl, CURLOPT_URL, xlxHostsFileURL.c_str());
if(filedes > 0 && (fp = fdopen(filedes,"wb")) != nullptr) {
curl_easy_setopt(curl, CURLOPT_URL, hostsFileURL.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
CURLcode res = curl_easy_perform(curl);
@ -58,9 +59,20 @@ std::string CXLXHostsFileDownloader::download(const std::string & xlxHostsFileUR
if(ok) {
outFileName = std::string(outFileNameBuf);
try
{
std::filesystem::copy_file(outFileName, hostFilePath, std::filesystem::copy_options::overwrite_existing);
}
catch(std::filesystem::filesystem_error& e)
{
ok = false;
CLog::logError("Failed to copy host file to %s: %s", hostFilePath.c_str(), e.what());
}
} else {
CLog::logError("Failed to download XLx Host file from %s", xlxHostsFileURL.c_str());
CLog::logWarning("Failed to download Host file from %s, using previous file", hostsFileURL.c_str());
}
return outFileName;
return ok;
}

@ -21,9 +21,9 @@
#include <string>
class CXLXHostsFileDownloader {
class CHostsFileDownloader {
public:
static std::string download(const std::string & xlxHostsFileURL);
static bool download(const std::string & hostsFileURL, const std::string & hostFilePath);
private:
static size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream);
};

@ -0,0 +1,211 @@
/*
* Copyright (c) 2024 by Geoffrey F4FXL / KC3FRA
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include <cassert>
#include "HostsFilesManager.h"
#include "HostFile.h"
#include "StringUtils.h"
#include "UDPReaderWriter.h"
#include "Log.h"
std::string CHostsFilesManager::m_hostFilesDirectory("");
std::string CHostsFilesManager::m_customFilesDirectory("");
bool CHostsFilesManager::m_dextraEnabled = false;
bool CHostsFilesManager::m_dcsEnabled = false;
bool CHostsFilesManager::m_dplusEnabled = false;
bool CHostsFilesManager::m_xlxEnabled = false;
std::string CHostsFilesManager::m_dextraUrl("");
std::string CHostsFilesManager::m_dcsUrl("");
std::string CHostsFilesManager::m_dplusUrl("");
std::string CHostsFilesManager::m_xlxUrl("");
CCacheManager * CHostsFilesManager::m_cache = nullptr;
CTimer CHostsFilesManager::m_downloadTimer(1000U, 60 * 60 * 24);
HostFileDownloadCallback CHostsFilesManager::m_downloadCallback = nullptr;
void CHostsFilesManager::setHostFilesDirectories(const std::string & hostFilesDir, const std::string & customHostFilesDir)
{
m_hostFilesDirectory.assign(hostFilesDir);
m_customFilesDirectory.assign(customHostFilesDir);
}
void CHostsFilesManager::setDownloadCallback(HostFileDownloadCallback callback)
{
m_downloadCallback = callback;
}
void CHostsFilesManager::setDextra(bool enabled, const std::string & url)
{
m_dextraEnabled = enabled;
m_dextraUrl.assign(url);
}
void CHostsFilesManager::setDPlus(bool enabled, const std::string & url)
{
m_dplusEnabled = enabled;
m_dplusUrl.assign(url);
}
void CHostsFilesManager::setDCS(bool enabled, const std::string & url)
{
m_dcsEnabled = enabled;
m_dcsUrl.assign(url);
}
void CHostsFilesManager::setXLX(bool enabled, const std::string & url)
{
m_xlxEnabled = enabled;
m_xlxUrl.assign(url);
}
void CHostsFilesManager::setCache(CCacheManager * cache)
{
assert(cache != nullptr);
m_cache = cache;
}
void CHostsFilesManager::clock(unsigned int ms)
{
m_downloadTimer.clock(ms);
if(m_downloadTimer.hasExpired()) {
CLog::logInfo("Downloading hosts files after %u hours", m_downloadTimer.getTimeout() / 3600U);
UpdateHostsAsync(); // call and forget
m_downloadTimer.start();
}
}
void CHostsFilesManager::setDownloadTimeout(unsigned int seconds)
{
m_downloadTimer.start(seconds);
}
bool CHostsFilesManager::UpdateHostsFromInternet()
{
CLog::logInfo("Updating hosts files from internet");
bool ret = true;
if(m_dextraEnabled && !m_dextraUrl.empty()) ret = m_downloadCallback(m_dextraUrl, m_hostFilesDirectory + "/" + DEXTRA_HOSTS_FILE_NAME) && ret;
if(m_dcsEnabled && !m_dcsUrl.empty()) ret = m_downloadCallback(m_dcsUrl, m_hostFilesDirectory + "/" + DCS_HOSTS_FILE_NAME) && ret;
if(m_dplusEnabled && !m_dplusUrl.empty()) ret = m_downloadCallback(m_dplusUrl, m_hostFilesDirectory + "/" + DPLUS_HOSTS_FILE_NAME) && ret;
if(m_xlxEnabled && !m_xlxUrl.empty()) ret = m_downloadCallback(m_xlxUrl, m_hostFilesDirectory + "/" + XLX_HOSTS_FILE_NAME) && ret;
if(!ret) CLog::logWarning("Some hosts files failed to downlaod");
CHostsFilesManager::loadReflectors(m_hostFilesDirectory);
return ret;
}
bool CHostsFilesManager::UpdateHostsFromLocal()
{
CHostsFilesManager::loadReflectors(m_customFilesDirectory);
return true;
}
bool CHostsFilesManager::UpdateHosts()
{
bool ret = UpdateHostsFromInternet();
UpdateHostsFromLocal() && ret;
return ret;
}
std::future<bool> CHostsFilesManager::UpdateHostsAsync()
{
auto fut = std::async(std::launch::async, UpdateHosts);
return fut;
}
void CHostsFilesManager::loadReflectors(const std::string & directory)
{
if (m_xlxEnabled) {
std::string fileName = directory + "/" + XLX_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DCS);
}
if (m_dplusEnabled) {
std::string fileName = directory + "/" + DPLUS_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DPLUS);
}
if (m_dextraEnabled) {
std::string fileName = directory + "/" + DEXTRA_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DEXTRA);
}
if (m_dcsEnabled) {
std::string fileName = directory + "/" + DCS_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DCS);
}
}
void CHostsFilesManager::loadReflectors(const std::string & hostFileName, DSTAR_PROTOCOL proto)
{
if(m_cache == nullptr) {
CLog::logDebug("HostsFilesManager cache not initilized");
return;
}
unsigned int count = 0U;
CHostFile hostFile(hostFileName, false);
for (unsigned int i = 0U; i < hostFile.getCount(); i++) {
std::string reflector = hostFile.getName(i);
in_addr address = CUDPReaderWriter::lookup(hostFile.getAddress(i));
bool lock = hostFile.getLock(i);
if (address.s_addr != INADDR_NONE) {
unsigned char* ucp = (unsigned char*)&address;
std::string addrText;
addrText = CStringUtils::string_format("%u.%u.%u.%u", ucp[0U] & 0xFFU, ucp[1U] & 0xFFU, ucp[2U] & 0xFFU, ucp[3U] & 0xFFU);
if (lock)
CLog::logInfo("Locking %s to %s", reflector.c_str(), addrText.c_str());
reflector.resize(LONG_CALLSIGN_LENGTH - 1U, ' ');
reflector += "G";
m_cache->updateGateway(reflector, addrText, proto, lock, true);
count++;
}
}
std::string protoString;
switch (proto)
{
case DP_DEXTRA:
protoString = "DExtra";
break;
case DP_DCS:
protoString = "DCS";
break;
case DP_DPLUS:
protoString = "DPlus";
break;
default:
// ???
break;
}
CLog::logInfo("Loaded %u of %u %s hosts from %s", count, hostFile.getCount(), protoString.c_str() , hostFileName.c_str());
}

@ -0,0 +1,70 @@
/*
* Copyright (c) 2024 by Geoffrey F4FXL / KC3FRA
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef HostsFilesManager_H
#define HostsFilesManager_H
#include <string>
#include <future>
#include "CacheManager.h"
#include "Timer.h"
#include "DStarDefines.h"
typedef bool (*HostFileDownloadCallback)(const std::string &, const std::string &);
class CHostsFilesManager {
public:
static void setHostFilesDirectories(const std::string & hostFilesDir, const std::string & customHostFilesDir);
static void setDownloadCallback(HostFileDownloadCallback downloadCallback);
static void setDextra(bool enabled, const std::string & dextraUrl);
static void setDCS(bool enabled, const std::string & dcsUrl);
static void setDPlus(bool enabled, const std::string & dplusUrl);
static void setXLX(bool enabled, const std::string & xlxUrl);
static void setCache(CCacheManager * cache);
static void clock(unsigned int ms);
static void setDownloadTimeout(unsigned int seconds);
static bool UpdateHostsFromInternet();
static bool UpdateHostsFromLocal();
static bool UpdateHosts();
static std::future<bool> UpdateHostsAsync();
private:
static std::string m_hostFilesDirectory;
static std::string m_customFilesDirectory;
static bool m_dextraEnabled;
static bool m_dcsEnabled;
static bool m_dplusEnabled;
static bool m_xlxEnabled;
static std::string m_dextraUrl;
static std::string m_dcsUrl;
static std::string m_dplusUrl;
static std::string m_xlxUrl;
static CCacheManager * m_cache;
static CTimer m_downloadTimer;
static HostFileDownloadCallback m_downloadCallback;
static void loadReflectors(const std::string & directory);
static void loadReflectors(const std::string & hostFileName, DSTAR_PROTOCOL proto);
};
#endif

@ -41,7 +41,6 @@
#include "Version.h"
#include "GitVersion.h"
#include "RepeaterProtocolHandlerFactory.h"
#include "XLXHostsFileDownloader.h"
#include "Log.h"
#include "LogFileTarget.h"
#include "LogConsoleTarget.h"
@ -50,6 +49,7 @@
#include "Daemon.h"
#include "APRSISHandlerThread.h"
#include "DummyAPRSHandlerThread.h"
#include "HostsFilesManager.h"
CDStarGatewayApp * CDStarGatewayApp::g_app = nullptr;
const std::string BANNER_1 = CStringUtils::string_format("%s Copyright (C) %s\n", FULL_PRODUCT_NAME.c_str(), VENDOR_NAME.c_str());
@ -68,8 +68,9 @@ int main(int argc, char *argv[])
signal(SIGILL, CDStarGatewayApp::sigHandlerFatal);
signal(SIGFPE, CDStarGatewayApp::sigHandlerFatal);
signal(SIGABRT, CDStarGatewayApp::sigHandlerFatal);
signal(SIGTERM, CDStarGatewayApp::sigHandler);
signal(SIGINT, CDStarGatewayApp::sigHandler);
signal(SIGTERM, CDStarGatewayApp::sigHandlerExit);
signal(SIGINT, CDStarGatewayApp::sigHandlerExit);
signal(SIGUSR1, CDStarGatewayApp::sigHandlerUSR);
setbuf(stdout, NULL);
if (2 != argc) {
@ -324,26 +325,36 @@ bool CDStarGatewayApp::createThread()
// Setup Dextra
TDextra dextraConfig;
m_config->getDExtra(dextraConfig);
CLog::logInfo("DExtra enabled: %d, max. dongles: %u", int(dextraConfig.enabled), dextraConfig.maxDongles);
CLog::logInfo("DExtra enabled: %d, max. dongles: %u, url: %s", int(dextraConfig.enabled), dextraConfig.maxDongles, dextraConfig.hostfileUrl.c_str());
m_thread->setDExtra(dextraConfig.enabled, dextraConfig.maxDongles);
// Setup DCS
TDCS dcsConfig;
m_config->getDCS(dcsConfig);
CLog::logInfo("DCS enabled: %d", int(dcsConfig.enabled));
CLog::logInfo("DCS enabled: %d, url: %s", int(dcsConfig.enabled), dcsConfig.hostfileUrl.c_str());
m_thread->setDCS(dcsConfig.enabled);
// Setup DPlus
TDplus dplusConfig;
m_config->getDPlus(dplusConfig);
CLog::logInfo("D-Plus enabled: %d, max. dongles: %u, login: %s", int(dplusConfig.enabled), dplusConfig.maxDongles, dplusConfig.login.c_str());
CLog::logInfo("D-Plus enabled: %d, max. dongles: %u, login: %s, url: %s", int(dplusConfig.enabled), dplusConfig.maxDongles, dplusConfig.login.c_str(), dplusConfig.hostfileUrl.c_str());
m_thread->setDPlus(dplusConfig.enabled, dplusConfig.maxDongles, dplusConfig.login);
// Setup XLX
TXLX xlxConfig;
m_config->getXLX(xlxConfig);
CLog::logInfo("XLX enabled: %d, Hosts file url: %s", int(xlxConfig.enabled), xlxConfig.url.c_str());
m_thread->setXLX(xlxConfig.enabled, xlxConfig.enabled ? CXLXHostsFileDownloader::download(xlxConfig.url) : "");
CLog::logInfo("XLX enabled: %d, Hosts file url: %s", int(xlxConfig.enabled), xlxConfig.hostfileUrl.c_str());
m_thread->setXLX(xlxConfig.enabled);
// Setup hostsfiles
THostsFiles hostsFilesConfig;
m_config->getHostsFiles(hostsFilesConfig);
CHostsFilesManager::setHostFilesDirectories(hostsFilesConfig.downloadedHostFiles, hostsFilesConfig.customHostsFiles);
CHostsFilesManager::setDownloadTimeout(3600 * hostsFilesConfig.downloadTimeout);
CHostsFilesManager::setDextra(dextraConfig.enabled, dextraConfig.hostfileUrl);
CHostsFilesManager::setDCS (dcsConfig.enabled, dcsConfig.hostfileUrl);
CHostsFilesManager::setDPlus (dplusConfig.enabled, dplusConfig.hostfileUrl);
CHostsFilesManager::setXLX (xlxConfig.enabled, xlxConfig.hostfileUrl);
// Setup Remote
TRemote remoteConfig;
@ -363,7 +374,7 @@ bool CDStarGatewayApp::createThread()
return true;
}
void CDStarGatewayApp::sigHandler(int sig)
void CDStarGatewayApp::sigHandlerExit(int sig)
{
CLog::logInfo("Caught signal : %s, shutting down gateway", strsignal(sig));
@ -385,6 +396,15 @@ void CDStarGatewayApp::sigHandlerFatal(int sig)
exit(3);
}
void CDStarGatewayApp::sigHandlerUSR(int sig)
{
if(sig == SIGUSR1) {
CLog::logInfo("Caught signal : %s, updating host files", strsignal(sig));
CHostsFilesManager::UpdateHostsAsync(); // call and forget
}
}
void CDStarGatewayApp::terminateHandler()
{
#ifdef DEBUG_DSTARGW

@ -38,6 +38,7 @@ public:
void run();
static void sigHandlerFatal(int sig);
static void sigHandler(int sig);
static void sigHandlerExit(int sig);
static void sigHandlerUSR(int sig);
static void terminateHandler();
};

@ -45,6 +45,7 @@ bool CDStarGatewayConfig::load()
ret = loadIrcDDB(cfg) && ret;
ret = loadRepeaters(cfg) && ret;
ret = loadPaths(cfg) && ret;
ret = loadHostsFiles(cfg) && ret;
ret = loadLog(cfg) && ret;
ret = loadAPRS(cfg) && ret;
ret = loadDextra(cfg) && ret;
@ -83,9 +84,9 @@ bool CDStarGatewayConfig::loadDaemon(const CConfig & cfg)
bool CDStarGatewayConfig::loadXLX(const CConfig & cfg)
{
bool ret = cfg.getValue("xlx", "enabled", m_xlx.enabled, true);
ret = cfg.getValue("xlx", "hostfileUrl", m_xlx.url, 0, 1024, "") && ret;
ret = cfg.getValue("xlx", "hostfileUrl", m_xlx.hostfileUrl, 0, 1024, "") && ret;
m_xlx.enabled = m_xlx.enabled && !m_xlx.url.empty();
m_xlx.enabled = m_xlx.enabled;
return ret;
}
@ -105,6 +106,7 @@ bool CDStarGatewayConfig::loadDextra(const CConfig & cfg)
{
bool ret = cfg.getValue("dextra", "enabled", m_dextra.enabled, true);
ret = cfg.getValue("dextra", "maxDongles", m_dextra.maxDongles, 1U, 5U, 5U) && ret;
ret = cfg.getValue("dextra", "hostfileUrl", m_dextra.hostfileUrl, 0, 1024, "") && ret;
return ret;
}
@ -113,6 +115,7 @@ bool CDStarGatewayConfig::loadDPlus(const CConfig & cfg)
bool ret = cfg.getValue("dplus", "enabled", m_dplus.enabled, true);
ret = cfg.getValue("dplus", "maxDongles", m_dplus.maxDongles, 1U, 5U, 5U) && ret;
ret = cfg.getValue("dplus", "login", m_dplus.login, 0, LONG_CALLSIGN_LENGTH, m_gateway.callsign) && ret;
ret = cfg.getValue("dplus", "hostfileUrl", m_dplus.hostfileUrl, 0, 1024, "") && ret;
m_dplus.enabled = m_dplus.enabled && !m_dplus.login.empty();
m_dplus.login = CUtils::ToUpper(m_dplus.login);
@ -123,6 +126,7 @@ bool CDStarGatewayConfig::loadDPlus(const CConfig & cfg)
bool CDStarGatewayConfig::loadDCS(const CConfig & cfg)
{
bool ret = cfg.getValue("dcs", "enabled", m_dcs.enabled, true);
ret = cfg.getValue("dcs", "hostfileUrl", m_dcs.hostfileUrl, 0, 1024, "") && ret;
return ret;
}
@ -202,6 +206,25 @@ bool CDStarGatewayConfig::loadPaths(const CConfig & cfg)
return ret;
}
bool CDStarGatewayConfig::loadHostsFiles(const CConfig & cfg)
{
bool ret = cfg.getValue("HostsFiles", "downloadedHostsFiles", m_hostsFiles.downloadedHostFiles, 0, 2048, "/usr/local/share/dstargateway.d/");
ret = cfg.getValue("HostsFiles", "customHostsfiles", m_hostsFiles.customHostsFiles, 0, 2048, "/usr/local/share/dstargateway.d/hostsfiles.d/");
ret = cfg.getValue("HostsFiles", "downloadTimer", m_hostsFiles.downloadTimeout, 24U, 0xffffffffU, 72U);
if(ret && m_hostsFiles.downloadedHostFiles[m_hostsFiles.downloadedHostFiles.length() - 1] != '/') {
m_hostsFiles.downloadedHostFiles.push_back('/');
}
if(ret && m_hostsFiles.customHostsFiles[m_hostsFiles.customHostsFiles.length() - 1] != '/') {
m_hostsFiles.downloadedHostFiles.push_back('/');
}
//TODO 20211226 check if directory are accessible
return ret;
}
bool CDStarGatewayConfig::loadRepeaters(const CConfig & cfg)
{
m_repeaters.clear();
@ -433,6 +456,11 @@ void CDStarGatewayConfig::getPaths(Tpaths & paths) const
paths = m_paths;
}
void CDStarGatewayConfig::getHostsFiles(THostsFiles & hostsFiles) const
{
hostsFiles = m_hostsFiles;
}
void CDStarGatewayConfig::getAPRS(TAPRS & aprs) const
{
aprs = m_aprs;

@ -82,6 +82,12 @@ typedef struct {
std::string dataDir;
} Tpaths;
typedef struct {
std::string downloadedHostFiles;
std::string customHostsFiles;
unsigned int downloadTimeout;
} THostsFiles;
typedef struct {
std::string logDir;
LOG_SEVERITY displayLevel;
@ -102,16 +108,19 @@ typedef struct {
typedef struct {
bool enabled;
unsigned int maxDongles;
std::string hostfileUrl;
} TDextra;
typedef struct {
bool enabled;
std::string login;
unsigned int maxDongles;
std::string hostfileUrl;
} TDplus;
typedef struct {
bool enabled;
std::string hostfileUrl;
} TDCS;
typedef struct {
@ -120,7 +129,7 @@ typedef struct {
typedef struct {
bool enabled;
std::string url;
std::string hostfileUrl;
} TXLX;
typedef struct {
@ -156,6 +165,7 @@ public:
unsigned int getRepeaterCount() const;
void getLog(TLog& log) const;
void getPaths(Tpaths & paths) const;
void getHostsFiles(THostsFiles & hostsFiles) const;
void getAPRS(TAPRS & aprs) const;
void getDExtra(TDextra & dextra) const;
void getDPlus(TDplus & dplus) const;
@ -176,6 +186,7 @@ private:
bool loadRepeaters(const CConfig & cfg);
bool loadLog(const CConfig & cfg);
bool loadPaths(const CConfig & cfg);
bool loadHostsFiles(const CConfig & cfg);
bool loadAPRS(const CConfig & cfg);
bool loadDextra(const CConfig & cfg);
bool loadDPlus(const CConfig & cfg);
@ -192,6 +203,7 @@ private:
std::string m_fileName;
TGateway m_gateway;
Tpaths m_paths;
THostsFiles m_hostsFiles;
TAPRS m_aprs;
TDextra m_dextra;
TDplus m_dplus;

@ -54,6 +54,8 @@
#include "Defs.h"
#include "Log.h"
#include "StringUtils.h"
#include "HostsFilesManager.h"
#include "HostsFileDownloader.h"
const std::string LOOPBACK_ADDRESS("127.0.0.1");
@ -88,7 +90,6 @@ m_dplusMaxDongles(0U),
m_dplusLogin(),
m_dcsEnabled(true),
m_xlxEnabled(true),
m_xlxHostsFileName(),
m_ccsEnabled(true),
m_ccsHost(),
m_infoEnabled(true),
@ -137,6 +138,10 @@ CDStarGatewayThread::~CDStarGatewayThread()
void* CDStarGatewayThread::Entry()
{
CHostsFilesManager::setCache(&m_cache);
CHostsFilesManager::setDownloadCallback(CHostsFileDownloader::download);
CHostsFilesManager::UpdateHosts();
// Truncate the old Links.log file
std::string fullName = m_logDir + "/" + LINKS_BASE_NAME + ".log";
if (!m_name.empty()) {
@ -218,9 +223,6 @@ void* CDStarGatewayThread::Entry()
}
}
loadGateways();
loadAllReflectors();
CG2Handler::setG2ProtocolHandlerPool(m_g2HandlerPool);
CG2Handler::setHeaderLogger(headerLogger);
@ -406,16 +408,16 @@ void* CDStarGatewayThread::Entry()
}
catch (std::exception& e) {
std::string message(e.what());
CLog::logFatal("Exception raised in the main thread - \"%s\"", message.c_str());
CLog::logFatal("Exception raised in the DStar Gateway thread - \"%s\"", message.c_str());
throw;
}
catch (...) {
CLog::logFatal("Unknown exception raised in the main thread");
CLog::logFatal("Unknown exception raised in the DStar Gateway thread");
throw;
}
#endif
CLog::logInfo("Stopping the ircDDB Gateway thread");
CLog::logInfo("Stopping the DStar Gateway thread");
// Unlink from all reflectors
CDExtraHandler::unlink();
@ -599,10 +601,9 @@ void CDStarGatewayThread::setDCS(bool enabled)
m_dcsEnabled = enabled;
}
void CDStarGatewayThread::setXLX(bool enabled, const std::string& xlxHostsFileName)
void CDStarGatewayThread::setXLX(bool enabled)
{
m_xlxEnabled = enabled;
m_xlxHostsFileName = xlxHostsFileName;
}
#ifdef USE_CCS
@ -1141,81 +1142,6 @@ void CDStarGatewayThread::processDD()
}
}
void CDStarGatewayThread::loadGateways()
{
std::string fileName = m_dataDir + "/" + GATEWAY_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DEXTRA);
}
void CDStarGatewayThread::loadAllReflectors()
{
if (m_xlxEnabled) {
loadReflectors(m_xlxHostsFileName, DP_DCS);
}
if (m_dplusEnabled) {
std::string fileName = m_dataDir + "/" + DPLUS_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DPLUS);
}
if (m_dextraEnabled) {
std::string fileName = m_dataDir + "/" + DEXTRA_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DEXTRA);
}
if (m_dcsEnabled) {
std::string fileName = m_dataDir + "/" + DCS_HOSTS_FILE_NAME;
loadReflectors(fileName, DP_DCS);
}
}
void CDStarGatewayThread::loadReflectors(std::string hostFileName, DSTAR_PROTOCOL proto)
{
unsigned int count = 0U;
CHostFile hostFile(hostFileName, false);
for (unsigned int i = 0U; i < hostFile.getCount(); i++) {
std::string reflector = hostFile.getName(i);
in_addr address = CUDPReaderWriter::lookup(hostFile.getAddress(i));
bool lock = hostFile.getLock(i);
if (address.s_addr != INADDR_NONE) {
unsigned char* ucp = (unsigned char*)&address;
std::string addrText;
addrText = CStringUtils::string_format("%u.%u.%u.%u", ucp[0U] & 0xFFU, ucp[1U] & 0xFFU, ucp[2U] & 0xFFU, ucp[3U] & 0xFFU);
if (lock)
CLog::logInfo("Locking %s to %s", reflector.c_str(), addrText.c_str());
reflector.resize(LONG_CALLSIGN_LENGTH - 1U, ' ');
reflector += "G";
m_cache.updateGateway(reflector, addrText, proto, lock, true);
count++;
}
}
std::string protoString;
switch (proto)
{
case DP_DEXTRA:
protoString = "DExtra";
break;
case DP_DCS:
protoString = "DCS";
break;
case DP_DPLUS:
protoString = "DPlus";
break;
default:
// ???
break;
}
CLog::logInfo("Loaded %u of %u %s hosts from %s", count, hostFile.getCount(), protoString.c_str() , hostFileName.c_str());
}
void CDStarGatewayThread::writeStatus()
{
std::string fullName = LINKS_BASE_NAME;

@ -17,8 +17,8 @@
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef IRCDDBGatewayThread_H
#define IRCDDBGatewayThread_H
#ifndef DStarGatewayThread_H
#define DStarGatewayThread_H
#include "DummyRepeaterProtocolHandler.h"
#include "IcomRepeaterProtocolHandler.h"
@ -61,7 +61,7 @@ public:
virtual void setDExtra(bool enabled, unsigned int maxDongles);
virtual void setDPlus(bool enabled, unsigned int maxDongles, const std::string& login);
virtual void setDCS(bool enabled);
virtual void setXLX(bool enabled, const std::string& fileName);
virtual void setXLX(bool enabled);
#ifdef USE_CCS
virtual void setCCS(bool enabled, const std::string& host);
#endif
@ -112,7 +112,6 @@ private:
std::string m_dplusLogin;
bool m_dcsEnabled;
bool m_xlxEnabled;
std::string m_xlxHostsFileName;
bool m_ccsEnabled;
std::string m_ccsHost;
bool m_infoEnabled;
@ -147,14 +146,6 @@ private:
void processG2();
void processDD();
void loadGateways();
void loadAllReflectors();
void loadReflectors(std::string hostFileName, DSTAR_PROTOCOL proto);
void loadDExtraReflectors(const std::string& fileName);
void loadDPlusReflectors(const std::string& fileName);
void loadDCSReflectors(const std::string& fileName);
void loadXLXReflectors();
void writeStatus();
void readStatusFiles();

@ -155,23 +155,32 @@ displayLevel= # defaults to info, valid values are trace, debug, info, warning
repeatThreshold=#defaults to 2, valid values are disbaled and 1 to 10. Prevents flooding of logs from repeated log messages.
[Paths]
data=/usr/local/share/dstargateway.d/ #Path where the data (hostfiles, audio files etc) can be found
data=/usr/local/share/dstargateway.d/ # Path where the data (audio files etc) can be found
[HostsFiles]
downloadedHostsFiles=/usr/local/share/dstargateway.d/ # Path where the downloaded host files are stored, make sure user dstar has write access
downloadTimer= # Redownload host files every X hours, must be greater than 24 hours. Defaults to 72 hours
customHostsfiles=/usr/local/share/dstargateway.d/hostfiles.d/ # Place your custom host files in this directory, this dir must be different from datadir.
# Any hosts found here will override same host in downloaded files
[DExtra]
enabled=true # There is no reason to disable this
maxDongles=5
hostfileUrl=http://www.pistar.uk/downloads/DExtra_Hosts.txt # URL where to download the host file on startup and every X minutes. Leave empty to disable.
[DPlus]
enabled=true # There is no reason to disable this
maxDongles=5
login= # defaults to gateway callsign
hostfileUrl=http://www.pistar.uk/downloads/DPlus_Hosts.txt # URL where to download the host file on startup and every X minutes. Leave empty to disable.
[DCS]
enabled=true # There is no reason to disable this
hostfileUrl=http://www.pistar.uk/downloads/DCS_Hosts.txt # URL where to download the host file on startup and every X minutes. Leave empty to disable.
[XLX]
enabled=true
hostfileUrl=http://xlxapi.rlx.lu/api.php?do=GetXLXDMRMaster
hostfileUrl=http://xlxapi.rlx.lu/api.php?do=GetXLXDMRMaster # URL where to download the host file on startup and every X minutes. Leave empty to disable.
[DRats]
enabled=false # Defaults to false. The program need to be compiled with DRats support for DRats to be actually enabled

@ -117,6 +117,7 @@ install : DStarGateway/dstargateway DGWRemoteControl/dgwremotecontrol
@mkdir -p $(BIN_DIR)
@mkdir -p $(LOG_DIR)
@chown dstar:dstar $(LOG_DIR)
@chown dstar:dstar $(DATA_DIR)
# Install data
$(MAKE) -C Data install

@ -65,6 +65,7 @@ All the features found in ircddbGateway are supposed to be working. Except the o
- DPlus, DExtra and G2 NAT Traversal using ircddb network as rendez-vous server. I.e. it is not required to open firewall ports for Callsign Routing or Gateway calls. however it is still recommended to do so. But NAT Traversal will bring more flexibility when operating on CGNAT (Mobile) Networks.
- Forward RSMS1A app messages from/to APRS-IS Network, yes you can send/receive messages to and from aprs. Yes, you can send messages to APRS stations and Vice Versa. Additionnally, part of the message is sent as Text Dat in the slow data. This allows you to read the message directly on your radio screen.
- Repeater Link status is sent to APRS-IS as a status frame
- Built-in download of hosts files
# 3. Building and installing
## 3.1. Initial setup
@ -134,6 +135,7 @@ To update host files, from within the source code directory, run
sudo make newhostfiles
sudo systemctl restart dstargateway.service
```
or juste configure and run the software and it will download latest host files.
# 4. Dashboard
@johnhays K7VE has developed a nice lightweight NodeJS dashboard. Code and instructions can be found on his [GitHub](https://github.com/johnhays/dsgwdashboard).
@ -156,6 +158,7 @@ The testing framwework used is Google Test.
# 6. Version History
## 6.1. Version 1.0
- [**Improvement**] Automatically download hosts files on startup. It is also possible to specify custom hosts files to override hosts from the internet. Needs some configuration adjustement, see example.cfg. ([#50](https://github.com/F4FXL/DStarGateway/issues/50))
- [**Improvement**] Limit log reporting ([#44](https://github.com/F4FXL/DStarGateway/issues/44))
- [**Improvement**] Improve CI to include all variants of build configurations ([#40](https://github.com/F4FXL/DStarGateway/issues/40))
- [**Bugfix**] Fix #43 Cache not updated when answering ircddb gateway is only conected to one network ([#43](https://github.com/F4FXL/DStarGateway/issues/43))

@ -0,0 +1,207 @@
/*
* Copyright (C) 2021-2024 by Geoffrey Merck F4FXL / KC3FRA
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include <gtest/gtest.h>
#include <unistd.h>
#include <string>
#include <stdlib.h>
#include <arpa/inet.h>
#include "HostsFilesManager.h"
#include "CacheManager.h"
#include "DStarDefines.h"
namespace HostsFilesManagerTests
{
class HostsFilesManager_UpdateHosts : public ::testing::Test
{
protected:
std::string m_internetPath;
std::string m_customPath;
CCacheManager * m_cache;
HostsFilesManager_UpdateHosts() :
m_internetPath(),
m_customPath(),
m_cache(nullptr)
{
}
void SetUp() override
{
char buf[2048];
auto size = ::readlink("/proc/self/exe", buf, 2048);
if(size > 0) {
std::string path(buf, size);
auto lastSlashPos = path.find_last_of('/');
path.resize(lastSlashPos);
m_internetPath = path + "/HostsFilesManager/internet/";
m_customPath = path + "/HostsFilesManager/custom/";
}
CHostsFilesManager::setDCS(false, "");
CHostsFilesManager::setDextra(false, "");
CHostsFilesManager::setXLX(false, "");
CHostsFilesManager::setDPlus(false, "");
if(m_cache != nullptr) delete m_cache;
m_cache = new CCacheManager();
}
static bool dummyDownload(const std::string & a, const std::string & b)
{
std::cout << a << std::endl << b;
return true;
}
};
TEST_F(HostsFilesManager_UpdateHosts, DExtraFromInternet)
{
CHostsFilesManager::setDextra(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet");
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("XRF123 G");
EXPECT_NE(gw, nullptr) << "DExtra host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "XRF123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DEXTRA) << "Protocol mismatch";
}
TEST_F(HostsFilesManager_UpdateHosts, DExtraFromInternetCustomOverride)
{
CHostsFilesManager::setDextra(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath);
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("XRF123 G");
EXPECT_NE(gw, nullptr) << "DExtra host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "XRF123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DEXTRA) << "Protocol mismatch";
}
TEST_F(HostsFilesManager_UpdateHosts, DCSFromInternet)
{
CHostsFilesManager::setDCS(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet");
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("DCS123 G");
EXPECT_NE(gw, nullptr) << "DCS host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "DCS123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch";
}
TEST_F(HostsFilesManager_UpdateHosts, DCSFromInternetCustomOverride)
{
CHostsFilesManager::setDCS(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath);
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("DCS123 G");
EXPECT_NE(gw, nullptr) << "DCS host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "DCS123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch";
}
TEST_F(HostsFilesManager_UpdateHosts, DPlusFromInternet)
{
CHostsFilesManager::setDPlus(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet");
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("REF123 G");
EXPECT_NE(gw, nullptr) << "DPlus host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "REF123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DPLUS) << "Protocol mismatch";
}
TEST_F(HostsFilesManager_UpdateHosts, DPlusFromInternetCustomOverride)
{
CHostsFilesManager::setDPlus(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath);
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("REF123 G");
EXPECT_NE(gw, nullptr) << "DPlus host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "REF123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DPLUS) << "Protocol mismatch";
}
TEST_F(HostsFilesManager_UpdateHosts, XLXFromInternet)
{
CHostsFilesManager::setXLX(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, "specify invalid custom path as we do not want to override hosts from the internet");
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("XLX123 G");
EXPECT_NE(gw, nullptr) << "XLX host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "XLX123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("1.1.1.1")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch";
}
TEST_F(HostsFilesManager_UpdateHosts, XLXFromInternetCustomOverride)
{
CHostsFilesManager::setXLX(true, "files are actually not downloaded, we want to to check that DEXtra Hosts are stored as DExtra");
CHostsFilesManager::setHostFilesDirectories(m_internetPath, m_customPath);
CHostsFilesManager::setDownloadCallback(HostsFilesManager_UpdateHosts::dummyDownload);
CHostsFilesManager::setCache(m_cache);
CHostsFilesManager::UpdateHosts();
auto gw = m_cache->findGateway("XLX123 G");
EXPECT_NE(gw, nullptr) << "XLX host not found";
EXPECT_STREQ(gw->getGateway().c_str(), "XLX123 G");
EXPECT_EQ(gw->getAddress().s_addr, ::inet_addr("2.2.2.2")) << "Address missmatch";
EXPECT_EQ(gw->getProtocol(), DP_DCS) << "Protocol mismatch";
}
}
Loading…
Cancel
Save

Powered by TurnKey Linux.