From 411728e375ea09197cacf8ad8f03a2abae8ca48b Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 17 Apr 2023 14:06:16 -0700 Subject: [PATCH 01/41] better dht-values def --- reflector/Reflector.cpp | 24 +++--- reflector/Reflector.h | 2 +- reflector/dht-values.h | 156 ++++++++++++++++++++++++++++++++++++ reflector/urfd-dht-values.h | 89 -------------------- 4 files changed, 169 insertions(+), 102 deletions(-) create mode 100644 reflector/dht-values.h delete mode 100644 reflector/urfd-dht-values.h diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index afe9505..b02f8f7 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -555,7 +555,7 @@ void CReflector::PutDHTPeers() ReleasePeers(); auto nv = std::make_shared(p); - nv->user_type.assign("urfd-peers-1"); + nv->user_type.assign(URFD_PEERS_1); nv->id = toUType(EUrfdValueID::Peers); node.putSigned( @@ -584,7 +584,7 @@ void CReflector::PutDHTClients() ReleaseClients(); auto nv = std::make_shared(c); - nv->user_type.assign("urfd-clients-1"); + nv->user_type.assign(URFD_CLIENTS_1); nv->id = toUType(EUrfdValueID::Clients); node.putSigned( @@ -613,7 +613,7 @@ void CReflector::PutDHTUsers() ReleaseUsers(); auto nv = std::make_shared(u); - nv->user_type.assign("urfd-users-1"); + nv->user_type.assign(URFD_USERS_1); nv->id = toUType(EUrfdValueID::Users); node.putSigned( @@ -633,11 +633,11 @@ void CReflector::PutDHTConfig() const std::string cs(g_Configure.GetString(g_Keys.names.callsign)); SUrfdConfig1 cfg; time(&cfg.timestamp); - cfg.cs.assign(cs); - cfg.ipv4.assign(g_Configure.GetString(g_Keys.ip.ipv4address)); - cfg.ipv6.assign(g_Configure.GetString(g_Keys.ip.ipv6address)); - cfg.mods.assign(g_Configure.GetString(g_Keys.modules.modules)); - cfg.tcmods.assign(g_Configure.GetString(g_Keys.modules.tcmodules)); + cfg.callsign.assign(cs); + cfg.ipv4addr.assign(g_Configure.GetString(g_Keys.ip.ipv4address)); + cfg.ipv6addr.assign(g_Configure.GetString(g_Keys.ip.ipv6address)); + cfg.modules.assign(g_Configure.GetString(g_Keys.modules.modules)); + cfg.transcodedmods.assign(g_Configure.GetString(g_Keys.modules.tcmodules)); cfg.url.assign(g_Configure.GetString(g_Keys.names.url)); cfg.email.assign(g_Configure.GetString(g_Keys.names.email)); cfg.country.assign(g_Configure.GetString(g_Keys.names.country)); @@ -663,11 +663,11 @@ void CReflector::PutDHTConfig() cfg.port[toUType(EUrfdPorts::urf)] = (uint16_t)g_Configure.GetUnsigned(g_Keys.urf.port); cfg.port[toUType(EUrfdPorts::ysf)] = (uint16_t)g_Configure.GetUnsigned(g_Keys.ysf.port); cfg.g3enabled = g_Configure.GetBoolean(g_Keys.g3.enable); - for (const auto m : cfg.mods) + for (const auto m : cfg.modules) cfg.description[m] = g_Configure.GetString(g_Keys.modules.descriptor[m-'A']); auto nv = std::make_shared(cfg); - nv->user_type.assign("urfd-config-1"); + nv->user_type.assign(URFD_CONFIG_1); nv->id = toUType(EUrfdValueID::Config); node.putSigned( @@ -696,7 +696,7 @@ void CReflector::GetDHTConfig(const std::string &cs) node.get( dht::InfoHash::get(cs), [](const std::shared_ptr &v) { - if (0 == v->user_type.compare("urfd-config-1")) + if (0 == v->user_type.compare(URFD_CONFIG_1)) { auto rdat = dht::Value::unpack(*v); if (rdat.timestamp > cfg.timestamp) @@ -717,7 +717,7 @@ void CReflector::GetDHTConfig(const std::string &cs) if (cfg.timestamp) { // if the get() call was successful and there is a nonzero timestamp, then do the update - g_GateKeeper.GetInterlinkMap()->Update(cfg.cs, cfg.mods, cfg.ipv4, cfg.ipv6, cfg.port[toUType(EUrfdPorts::urf)], cfg.tcmods); + g_GateKeeper.GetInterlinkMap()->Update(cfg.callsign, cfg.modules, cfg.ipv4addr, cfg.ipv6addr, cfg.port[toUType(EUrfdPorts::urf)], cfg.transcodedmods); g_GateKeeper.ReleaseInterlinkMap(); } else diff --git a/reflector/Reflector.h b/reflector/Reflector.h index 9f3c925..5d8fb52 100644 --- a/reflector/Reflector.h +++ b/reflector/Reflector.h @@ -27,7 +27,7 @@ #include "PacketStream.h" #ifndef NO_DHT -#include "urfd-dht-values.h" +#include "dht-values.h" #endif diff --git a/reflector/dht-values.h b/reflector/dht-values.h new file mode 100644 index 0000000..8919a2e --- /dev/null +++ b/reflector/dht-values.h @@ -0,0 +1,156 @@ +// Copyright © 2022 Thomas A. Early, N7TAE +// +// ---------------------------------------------------------------------------- +// This is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// with this software. If not, see . +// ---------------------------------------------------------------------------- + +#pragma once + +#include + +//#define USE_MREFD_VALUES +#define USE_URFD_VALUES + +/* HELPERS */ +template constexpr auto toUType(E enumerator) noexcept +{ + return static_cast>(enumerator); +} // Item #10 in "Effective Modern C++", by Scott Meyers, O'REILLY + +// every value type needs a const user_type and a timestamp +struct SDhtValueBase +{ + SDhtValueBase(const std::string &s) : user_type(s) {} + std::string user_type; + std::time_t timestamp; +}; + +#ifdef USE_MREFD_VALUES + +// user_type for mrefd values +#define MREFD_USERS_1 "mrefd-users-1" +#define MREFD_PEERS_1 "mrefd-peers-1" +#define MREFD_CONFIG_1 "mrefd-config-1" +#define MREFD_CLIENTS_1 "mrefd-clients-1" + +// dht::Value ids of the different parts of the document +// can be assigned any unsigned value except 0 +enum class EMrefdValueID : uint64_t { Config=1, Peers=2, Clients=3, Users=4 }; + +using MrefdPeerTuple = std::tuple; +enum class EMrefdPeerFields { Callsign, Modules, ConnectTime }; +struct SMrefdPeers1 : public SDhtValueBase +{ + SMrefdPeers1() : SDhtValueBase(MREFD_PEERS_1) {} + unsigned int sequence; + std::list list; + + MSGPACK_DEFINE(timestamp, sequence, list) +}; + +using MrefdClientTuple = std::tuple; +enum class EMrefdClientFields { Callsign, Ip, Module, ConnectTime, LastHeardTime }; +struct SMrefdClients1 : public SDhtValueBase +{ + SMrefdClients1() : SDhtValueBase(MREFD_CLIENTS_1) {} + unsigned int sequence; + std::list list; + + MSGPACK_DEFINE(timestamp, sequence, list) +}; + +using MrefdUserTuple = std::tuple; +enum class EMrefdUserFields { Source, Destination, Reflector, LastHeardTime }; +struct SMrefdUsers1 : public SDhtValueBase +{ + SMrefdUsers1() : SDhtValueBase(MREFD_USERS_1) {} + unsigned int sequence; + std::list list; + + MSGPACK_DEFINE(timestamp, sequence, list) +}; + +struct SMrefdConfig1 : public SDhtValueBase +{ + SMrefdConfig1() : SDhtValueBase(MREFD_CONFIG_1) {} + std::string callsign, ipv4addr, ipv6addr, modules, encryptedmods, url, email, sponsor, country, version; + uint16_t port; + + MSGPACK_DEFINE(timestamp, callsign, ipv4addr, ipv6addr, modules, encryptedmods, url, email, sponsor, country, version, port) +}; + +#endif + +#ifdef USE_URFD_VALUES + +#define URFD_PEERS_1 "urfd-peers-1" +#define URFD_USERS_1 "urfd-users-1" +#define URFD_CONFIG_1 "urfd-config-1" +#define URFD_CLIENTS_1 "urfd-clients-1" + +enum class EUrfdValueID : uint64_t { Config=1, Peers=2, Clients=3, Users=4 }; + +using UrfdPeerTuple = std::tuple; +enum class EUrfdPeerFields { Callsign, Modules, ConnectTime }; +struct SUrfdPeers1 : public SDhtValueBase +{ + SUrfdPeers1() : SDhtValueBase(URFD_PEERS_1) {} + unsigned int sequence; + std::list list; + + MSGPACK_DEFINE(timestamp, sequence, list) +}; + +using UrfdClientTuple = std::tuple; +enum class EUrfdClientFields { Callsign, Ip, Module, ConnectTime, LastHeardTime }; +struct SUrfdClients1 : public SDhtValueBase +{ + SUrfdClients1() : SDhtValueBase(URFD_CLIENTS_1) {} + unsigned int sequence; + std::list list; + + MSGPACK_DEFINE(timestamp, sequence, list) +}; + +using UrfdUserTuple = std::tuple; +enum class EUrfdUserFields { Callsign, ViaNode, OnModule, ViaPeer, LastHeardTime }; +struct SUrfdUsers1 : public SDhtValueBase +{ + SUrfdUsers1() : SDhtValueBase(URFD_USERS_1) {} + unsigned int sequence; + std::list list; + + MSGPACK_DEFINE(timestamp, sequence, list) +}; + +// 'SIZE' has to be last value for these scoped enums +enum class EUrfdPorts : unsigned { dcs, dextra, dmrplus, dplus, m17, mmdvm, nxdn, p25, urf, ysf, SIZE }; +enum class EUrfdAlMod : unsigned { nxdn, p25, ysf, SIZE }; +enum class EUrfdTxRx : unsigned { rx, tx, SIZE }; +enum class EUrfdRefId : unsigned { nxdn, p25, SIZE }; +struct SUrfdConfig1 : public SDhtValueBase +{ + SUrfdConfig1() : SDhtValueBase(URFD_CONFIG_1) {} + std::string callsign, ipv4addr, ipv6addr, modules, transcodedmods, url, email, sponsor, country, version; + std::array port; + std::array almod; + std::array ysffreq; + std::array refid; + std::unordered_map description; + bool g3enabled; + + MSGPACK_DEFINE(timestamp, callsign, ipv4addr, ipv6addr, modules, transcodedmods, url, email, sponsor, country, version, almod, ysffreq, refid, g3enabled, port, description) +}; + +#endif diff --git a/reflector/urfd-dht-values.h b/reflector/urfd-dht-values.h deleted file mode 100644 index ee147e6..0000000 --- a/reflector/urfd-dht-values.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright © 2023 Thomas A. Early, N7TAE -// -// ---------------------------------------------------------------------------- -// This file is part of urfd. -// -// M17Refd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// M17Refd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// with this software. If not, see . -// ---------------------------------------------------------------------------- - -#pragma once - -#include - -/* HELPERS */ -#ifndef TO_U_TYPE_DEF -#define TO_U_TYPE_DEF -template constexpr auto toUType(E enumerator) noexcept -{ - return static_cast>(enumerator); -} // Item #10 in "Effective Modern C++", by Scott Meyers, O'REILLY -#endif - -enum class EUrfdValueID : uint64_t { Config=1, Peers=2, Clients=3, Users=4 }; - -/* PEERS */ -using UrfdPeerTuple = std::tuple; -enum class EUrfdPeerFields { Callsign, Modules, ConnectTime }; -struct SUrfdPeers1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; - -/* CLIENTS */ -using UrfdClientTuple = std::tuple; -enum class EUrfdClientFields { Callsign, Ip, Module, ConnectTime, LastHeardTime }; -struct SUrfdClients1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; - -/* USERS */ -using UrfdUserTuple = std::tuple; -enum class EUrfdUserFields { Callsign, ViaNode, OnModule, ViaPeer, LastHeardTime }; -struct SUrfdUsers1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; - -/* CONFIGURATION */ -// 'SIZE' has to be last for these scoped enums -enum class EUrfdPorts : unsigned { dcs, dextra, dmrplus, dplus, m17, mmdvm, nxdn, p25, urf, ysf, SIZE }; -enum class EUrfdAlMod : unsigned { nxdn, p25, ysf, SIZE }; -enum class EUrfdTxRx : unsigned { rx, tx, SIZE }; -enum class EUrfdRefId : unsigned { nxdn, p25, SIZE }; -struct SUrfdConfig1 -{ - std::time_t timestamp; - std::string cs, ipv4, ipv6, mods, tcmods, url, email, sponsor, country, version; - std::array port; - std::array almod; - std::array ysffreq; - std::array refid; - std::unordered_map description; - bool g3enabled; - - MSGPACK_DEFINE(timestamp, cs, ipv4, ipv6, mods, tcmods, url, email, sponsor, country, version, almod, ysffreq, refid, g3enabled, port, description) -}; From 9b70c91c4240766666b21b2a8e1e8f8c59a92cf1 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Tue, 18 Apr 2023 10:13:07 -0700 Subject: [PATCH 02/41] don't need value base class --- reflector/dht-values.h | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/reflector/dht-values.h b/reflector/dht-values.h index 8919a2e..f0fed6f 100644 --- a/reflector/dht-values.h +++ b/reflector/dht-values.h @@ -19,7 +19,7 @@ #include -//#define USE_MREFD_VALUES +#define USE_MREFD_VALUES #define USE_URFD_VALUES /* HELPERS */ @@ -28,14 +28,6 @@ template constexpr auto toUType(E enumerator) noexcept return static_cast>(enumerator); } // Item #10 in "Effective Modern C++", by Scott Meyers, O'REILLY -// every value type needs a const user_type and a timestamp -struct SDhtValueBase -{ - SDhtValueBase(const std::string &s) : user_type(s) {} - std::string user_type; - std::time_t timestamp; -}; - #ifdef USE_MREFD_VALUES // user_type for mrefd values @@ -50,9 +42,9 @@ enum class EMrefdValueID : uint64_t { Config=1, Peers=2, Clients=3, Users=4 }; using MrefdPeerTuple = std::tuple; enum class EMrefdPeerFields { Callsign, Modules, ConnectTime }; -struct SMrefdPeers1 : public SDhtValueBase +struct SMrefdPeers1 { - SMrefdPeers1() : SDhtValueBase(MREFD_PEERS_1) {} + std::time_t timestamp; unsigned int sequence; std::list list; @@ -61,9 +53,9 @@ struct SMrefdPeers1 : public SDhtValueBase using MrefdClientTuple = std::tuple; enum class EMrefdClientFields { Callsign, Ip, Module, ConnectTime, LastHeardTime }; -struct SMrefdClients1 : public SDhtValueBase +struct SMrefdClients1 { - SMrefdClients1() : SDhtValueBase(MREFD_CLIENTS_1) {} + std::time_t timestamp; unsigned int sequence; std::list list; @@ -72,18 +64,18 @@ struct SMrefdClients1 : public SDhtValueBase using MrefdUserTuple = std::tuple; enum class EMrefdUserFields { Source, Destination, Reflector, LastHeardTime }; -struct SMrefdUsers1 : public SDhtValueBase +struct SMrefdUsers1 { - SMrefdUsers1() : SDhtValueBase(MREFD_USERS_1) {} + std::time_t timestamp; unsigned int sequence; std::list list; MSGPACK_DEFINE(timestamp, sequence, list) }; -struct SMrefdConfig1 : public SDhtValueBase +struct SMrefdConfig1 { - SMrefdConfig1() : SDhtValueBase(MREFD_CONFIG_1) {} + std::time_t timestamp; std::string callsign, ipv4addr, ipv6addr, modules, encryptedmods, url, email, sponsor, country, version; uint16_t port; @@ -103,9 +95,9 @@ enum class EUrfdValueID : uint64_t { Config=1, Peers=2, Clients=3, Users=4 }; using UrfdPeerTuple = std::tuple; enum class EUrfdPeerFields { Callsign, Modules, ConnectTime }; -struct SUrfdPeers1 : public SDhtValueBase +struct SUrfdPeers1 { - SUrfdPeers1() : SDhtValueBase(URFD_PEERS_1) {} + std::time_t timestamp; unsigned int sequence; std::list list; @@ -114,9 +106,9 @@ struct SUrfdPeers1 : public SDhtValueBase using UrfdClientTuple = std::tuple; enum class EUrfdClientFields { Callsign, Ip, Module, ConnectTime, LastHeardTime }; -struct SUrfdClients1 : public SDhtValueBase +struct SUrfdClients1 { - SUrfdClients1() : SDhtValueBase(URFD_CLIENTS_1) {} + std::time_t timestamp; unsigned int sequence; std::list list; @@ -125,9 +117,9 @@ struct SUrfdClients1 : public SDhtValueBase using UrfdUserTuple = std::tuple; enum class EUrfdUserFields { Callsign, ViaNode, OnModule, ViaPeer, LastHeardTime }; -struct SUrfdUsers1 : public SDhtValueBase +struct SUrfdUsers1 { - SUrfdUsers1() : SDhtValueBase(URFD_USERS_1) {} + std::time_t timestamp; unsigned int sequence; std::list list; @@ -139,9 +131,9 @@ enum class EUrfdPorts : unsigned { dcs, dextra, dmrplus, dplus, m17, mmdvm, nxdn enum class EUrfdAlMod : unsigned { nxdn, p25, ysf, SIZE }; enum class EUrfdTxRx : unsigned { rx, tx, SIZE }; enum class EUrfdRefId : unsigned { nxdn, p25, SIZE }; -struct SUrfdConfig1 : public SDhtValueBase +struct SUrfdConfig1 { - SUrfdConfig1() : SDhtValueBase(URFD_CONFIG_1) {} + std::time_t timestamp; std::string callsign, ipv4addr, ipv6addr, modules, transcodedmods, url, email, sponsor, country, version; std::array port; std::array almod; From b6718437de17a06b28367afeda88392be69fafe4 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 29 May 2023 05:56:28 -0700 Subject: [PATCH 03/41] README --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index aaa502f..4187b39 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Universal, Multi-protocol Digital Voice Reflector -The URF Multiprotocol Gateway Reflector Server, ***urfd***, is part of the software system for a Digital Voice Network. The sources are published under GPL Licenses. +The URF Multi-protocol Gateway Reflector Server, ***urfd***, is part of the software system for a Digital Voice Network. The sources are published under GPL Licenses. ## Introduction -This will build a new kind of digital voice reflector. A *urfd* supports DStar protocols (DPlus, DCS, DExtra and G3) DMR protocols (MMDVMHost, DMR+ and NXDN), M17, YSF, P25 (using IMBE) and USRP (Allstar). A key part of this is the hybrid transcoder, [tcd](https://github.com/n7tae/tcd), which is in a seperate repository. You can't interlink urfd with xlxd. This reflector can be built without a transcoder, but clients will only hear other clients using the same codec. Please note that currently, urfd only supports the tcd transcoder when run locally. As a local device, urfd and tcd uses UNIX DGRAM sockets for interprocess communications. These kernel-base sockets are signifantly faster than conventional UDP/IP sockets. It should be noted that tcd supports DVSI-3003 nad DVSI-3000 devices, which it uses for AMBE vocoding. +This will build a new kind of digital voice reflector. A *urfd* supports DStar protocols (DPlus, DCS, DExtra and G3) DMR protocols (MMDVMHost, DMR+ and NXDN), M17, YSF, P25 (using IMBE) and USRP (Allstar). A key part of this is the hybrid transcoder, [tcd](https://github.com/n7tae/tcd), which is in a separate repository. You can't interlink urfd with xlxd. This reflector can be built without a transcoder, but clients will only hear other clients using the same codec. Please note that currently, urfd only supports the tcd transcoder when run locally. As a local device, urfd and tcd uses UNIX DGRAM sockets for inter-process communications. These kernel-base sockets are signifantly faster than conventional UDP/IP sockets. It should be noted that tcd supports DVSI-3003 nad DVSI-3000 devices, which it uses for AMBE vocoding. This build support *dual-stack* operation, so the server on which it's running, must have both an IPv4 and IPv6 routable address if you are going to configure a dual-stack reflector. @@ -13,12 +13,12 @@ There are many improvements previous multi-mode reflectors: - Nearly all std::vector containers have been replaced with more appropriate containers. - No classes are derived from any standard containers. -- For concurancy, *i.e.*, thread management, the standard thread (std::thread) library calls have been replaced with std::future. +- For concurrency, *i.e.*, thread management, the standard thread (std::thread) library calls have been replaced with std::future. - Managed memory, std::unique_ptr and std::shared_ptr, is used replacing the need for calls to *new* and *delete*. -- Your reflector can be configured with up to 26 modules, *A* through *Z* and as few as one module. For other choices, the configure modules don't have to be contigious. For example, you could configure modules A, B, C and E. -- An integraded P25 Reflector with software imbe vocoder. +- Your reflector can be configured with up to 26 modules, *A* through *Z* and as few as one module. For other choices, the configure modules don't have to be contiguous. For example, you could configure modules A, B, C and E. +- An integrated P25 Reflector with software imbe vocoder. - An integrated NXDN Reflector -- An inegrated USRP Reflector +- An integrated USRP Reflector Only systemd-based operating systems are supported. Debian or Ubuntu is recommended. If you want to install this on a non-systemd based OS, you are on your own. Finally, this repository is designed so that you don't have to modify any file in the repository when you build your system. Any file you need to modify to properly configure your reflector will be a file you copy from you locally cloned repo. This makes it easier to update the source code when this repository is updated. Follow the instructions below to build your transcoding URF reflector. @@ -33,7 +33,7 @@ The packages which are described in this document are designed to install server Below are instructions to build a URF reflector. If you are planning on an URF reflector without a transcoder, you can help your users by naming modules with names that suggest which protocol is welcome. You name modules in the config.inc.php file mentioned below. -The transcoder is in a seperate repository, but you will build, install and monitor the transcoder and reflector from two different scripts, *rconfig* and *radmin* in this repository. You *should* look over the README.md file in the tcd repository to understand the transcoder. +The transcoder is in a separate repository, but you can install and monitor the transcoder and reflector from a script, *radmin* in this repository. You *should* look over the README.md file in the tcd repository to understand the transcoder. ### After a clean installation of Debian make sure to run update and upgrade @@ -51,9 +51,9 @@ sudo apt install build-essential sudo apt install nlohmann-json3-dev ``` -### DVIN support (optional, but highly recommended) +### Ham-DHT support (optional, but highly recommended) -**DVIN**, the Digital Voice Information Network, is implemented using a distributed hash table provided by OpenDHT. +**Ham-DHT**, a DHT network for hams, is implemented using a distributed hash table provided by OpenDHT. OpenDHT is available [here](https://github./com/savoirfairelinux/opendht.git). Building and installing instructions are in the [OpenDHT Wiki](https://github.com/savoirfairelinux/opendht/wiki/Build-the-library). Pascal support and proxy-server support (RESTinio) is not required for urfd and so can be considered optional. With this in mind, this should work on Debian/Ubuntu-based systems: From 70cc0107eeb75bf3dee73bba4b97489e7fd572d1 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 2 Jun 2023 09:41:19 -0700 Subject: [PATCH 04/41] doc improvements --- README.md | 7 ++----- config/urfd.ini | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4187b39..f1b2b10 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,10 @@ sudo apt update sudo apt upgrade ``` -### Required packages (some of these will probably already be installed) +### Required packages (some of these may already be installed) ```bash -sudo apt install git -sudo apt install apache2 php5 -sudo apt install build-essential -sudo apt install nlohmann-json3-dev +sudo apt install git apache2 php5 build-essential nlohmann-json3-dev libcurl4-gnutls-dev ``` ### Ham-DHT support (optional, but highly recommended) diff --git a/config/urfd.ini b/config/urfd.ini index f16d228..0aee3ac 100644 --- a/config/urfd.ini +++ b/config/urfd.ini @@ -3,7 +3,7 @@ # Do not use quotes, unless in a comment! [Names] -Callsign = URF??? # where ? is A-Z or 0-9 +Callsign = URF??? # where ? is A-Z or 0-9. NO EXCEPTIONS! SysopEmail = me@somewhere.com From 3808dce653f3534be5d58f52a3a37c471aa2f653 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 23 Oct 2023 17:45:45 -0700 Subject: [PATCH 05/41] README spelling typos --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f1b2b10..91c2661 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The URF Multi-protocol Gateway Reflector Server, ***urfd***, is part of the soft ## Introduction -This will build a new kind of digital voice reflector. A *urfd* supports DStar protocols (DPlus, DCS, DExtra and G3) DMR protocols (MMDVMHost, DMR+ and NXDN), M17, YSF, P25 (using IMBE) and USRP (Allstar). A key part of this is the hybrid transcoder, [tcd](https://github.com/n7tae/tcd), which is in a separate repository. You can't interlink urfd with xlxd. This reflector can be built without a transcoder, but clients will only hear other clients using the same codec. Please note that currently, urfd only supports the tcd transcoder when run locally. As a local device, urfd and tcd uses UNIX DGRAM sockets for inter-process communications. These kernel-base sockets are signifantly faster than conventional UDP/IP sockets. It should be noted that tcd supports DVSI-3003 nad DVSI-3000 devices, which it uses for AMBE vocoding. +This will build a new kind of digital voice reflector. A *urfd* supports DStar protocols (DPlus, DCS, DExtra and G3) DMR protocols (MMDVMHost, DMR+ and NXDN), M17, YSF, P25 (using IMBE) and USRP (Allstar). A key part of this is the hybrid transcoder, [tcd](https://github.com/n7tae/tcd), which is in a separate repository. You can't interlink urfd with xlxd. This reflector can be built without a transcoder, but clients will only hear other clients using the same codec. Please note that currently, urfd only supports the tcd transcoder when run locally. As a local device, urfd and tcd uses UNIX DGRAM sockets for inter-process communications. These kernel-base sockets are significantly faster than conventional UDP/IP sockets. It should be noted that tcd supports DVSI-3003 nad DVSI-3000 devices, which it uses for AMBE vocoding. This build support *dual-stack* operation, so the server on which it's running, must have both an IPv4 and IPv6 routable address if you are going to configure a dual-stack reflector. @@ -24,7 +24,7 @@ Only systemd-based operating systems are supported. Debian or Ubuntu is recommen ## Usage -The packages which are described in this document are designed to install server software which is used for the D-Star network infrastructure. It requires a 24/7 internet connection which can support up to three transcoded modules and up to 23 more untranscoded modules to connect repeaters and hot-spot dongles! +The packages which are described in this document are designed to install server software which is used for the D-Star network infrastructure. It requires a 24/7 internet connection which can support up to three transcoded modules and up to 23 more untranscoded modules to connect repeaters and hot-spot dongles. - The server can build a reflector that support IPv4, IPv6 or both (dual stack). - The public IP addresses should have a DNS record which must be published in the common host files. @@ -45,7 +45,7 @@ sudo apt upgrade ### Required packages (some of these may already be installed) ```bash -sudo apt install git apache2 php5 build-essential nlohmann-json3-dev libcurl4-gnutls-dev +sudo apt install git apache2 php build-essential nlohmann-json3-dev libcurl4-gnutls-dev ``` ### Ham-DHT support (optional, but highly recommended) @@ -90,12 +90,12 @@ This will create seven files: 1. The `urfd.mk` file contains compile-time options for *urfd*. If you change the `BINDIR`, you'll need to update how `urfd.service` starts *urfd*. 2. The `urfd.ini` file contains the run-time options for *urfd* and will be discussed below. 3. The `urfd.blacklist` file defines callsigns that are blocked from linking or transmitting. -4. The `urfd.whitelist` file defines callsigns that are allowed to link and transmit. Both of these files support the astrisk as a wild-card. The supplied blacklist and whitelist file are empty, which will allow any callsign to link and transmit, blocking no one. Both files support a limited wildcard feature. +4. The `urfd.whitelist` file defines callsigns that are allowed to link and transmit. Both of these files support the asterisk as a wild-card. The supplied blacklist and whitelist file are empty, which will allow any callsign to link and transmit, blocking no one. Both files support a limited wildcard feature. 5. The `urfd.interlink` file defines possible Brandmeister and URF linking. 6. The `urfd.terminal` file defines operations for Icom's Terminal and Access Point mode, sometimes called *G3*. This protocol requires significantly higher connection resources than any other mode, so it is possible to build a URF reflector without G3 support. 7. The `urfd.service` file is a systemd file that will start and stop *urfd*. Importantly, it contains the only reference to where the *urfd* ini file is located. Be sure to set a fully qualified path to your urfd.ini file on the `ExecStart` line. -You can acutally put the blacklist, whitelist, interlink, terminal and ini file anyplace and even rename them. Just make sure your ini file and service file have the proper, fully-qualified paths. The service file and the mk file need to remain in your `urfd/reflector` directory. +You can actually put the blacklist, whitelist, interlink, terminal and ini file anyplace and even rename them. Just make sure your ini file and service file have the proper, fully-qualified paths. The service file and the mk file need to remain in your `urfd/reflector` directory. When you are done with the configuration files and ready to start the installation process, you can return to the main repository directory: @@ -114,7 +114,7 @@ Use your favorite text editor to set your run-time configuration in your copy of There are only a few things that need to be specified. Most important are, the reflector callsign and the IP addresses for the IPv4 and IPv6 listen ports and a transcoder port, if there is a transcoder. Dual-stack operation is enabled by specifying both an IPv4 and IPv6 address. IPv4-only single stack can be specified by leaving the IPv6 address undefined. -You can configure any modules, from **A** to **Z**. They don't have to be contigious. If your reflector is configured with a transcoder, you can specify which configured modules will be transcoded. Up to three modules can be transcoded if you have the necessary hardware. +You can configure any modules, from **A** to **Z**. They don't have to be contiguous. If your reflector is configured with a transcoder, you can specify which configured modules will be transcoded. Up to three modules can be transcoded if you have the necessary hardware. Three protocols, BrandMeister, G3 and USRP should be disabled if you aren't going to use them. @@ -139,20 +139,20 @@ If you want to create listen-only clients, but you don't need a configured read/ ### Helper apps -There are two, very useful helper applications, *inicheck* and *dbutil*. Both apps will show you a usage message if you execuate them without any arguments. +There are two, very useful helper applications, *inicheck* and *dbutil*. Both apps will show you a usage message if you execute them without any arguments. -The *inicheck* app will use the exact same code that urfd uses to validate your `urfd.ini` file. Do `./inicheck -q mrefd.ini` to check your infile for errors. If you see any messages containing `ERROR`, that means that *urfd* won't start. You'll have to fix the errors described in the message(s). If you only see messages containing `WARNING`, *urfd* will start, but it may not perform as expected. You will have to decide if the warning should be fixed. If you don't see any messages, it means that your ini file is syntactly correct. +The *inicheck* app will use the exact same code that urfd uses to validate your `urfd.ini` file. Do `./inicheck -q mrefd.ini` to check your infile for errors. If you see any messages containing `ERROR`, that means that *urfd* won't start. You'll have to fix the errors described in the message(s). If you only see messages containing `WARNING`, *urfd* will start, but it may not perform as expected. You will have to decide if the warning should be fixed. If you don't see any messages, it means that your ini file is syntactically correct. -The *dbutil* app can be used for serveral tasks relating to the three databases that *urfd* uses. The usage is: `./dbutil DATABASE SOURCE ACTION INIFILE`, where: +The *dbutil* app can be used for several tasks relating to the three databases that *urfd* uses. The usage is: `./dbutil DATABASE SOURCE ACTION INIFILE`, where: - DATABASE is "dmr", "nxdn" or "ysf" - SOURCE is "html" or "file" - ACTION is "parse" or "errors" -- INIFLILE is the path to the infile that defines the location of the http and file sources for these three databases. +- INIFILE is the path to the infile that defines the location of the http and file sources for these three databases. One at a time, *dbutil* can work with any of the three DATABASEs. It can read either the http or the file SOURCE. It can either show you the data entries that are syntactically correct or incorrect (ACTION). ### Installing your system -After you have written your configutation files, you can install your system: +After you have written your configuration files, you can install your system: ```bash ./radmin @@ -160,7 +160,7 @@ After you have written your configutation files, you can install your system: You can use this interactive shell script to install and uninstall your system. This can also perform other tasks like restarting the reflector or transcoder process, or be used to view the reflector or transcoder log in real time. -### Stoping and starting the services manually +### Stopping and starting the services manually ```bash sudo systemctl stop urfd # (or xrfd) @@ -208,11 +208,11 @@ UDP port 62030 (MMDVM protocol) ## YSF Master Server Pay attention, the URF Server acts as an YSF Master, which provides 26 wires-x rooms. -It has nothing to do with the regular YSFReflector network, hence you don’t need to register your URF at ysfreflector.de ! +It has nothing to do with the regular YSFReflector network, hence you don’t need to register your URF at ysfreflector.de! ## To-dos -I will eventually support a remote transcoder option, so that you can, for example, run *urfd* in a data center, and then run the transcoder somewhere you have physical access so you can plug in your AMBE vocoders. I don't recommend this as it will add unnessary and variable latency to your reflector. +I will eventually support a remote transcoder option, so that you can, for example, run *urfd* in a data center, and then run the transcoder somewhere you have physical access so you can plug in your AMBE vocoders. I don't recommend this as it will add unnecessary and variable latency to your reflector. A new dashboard is on the to-do list! From 6aa57a784d1c4ae75b9c3570595f3fe76aaf27d9 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Sun, 19 Nov 2023 10:46:01 -0700 Subject: [PATCH 06/41] minor correction for M17 protocol --- reflector/M17Protocol.cpp | 6 ++---- reflector/Main.cpp | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/reflector/M17Protocol.cpp b/reflector/M17Protocol.cpp index 2516389..d51dfbd 100644 --- a/reflector/M17Protocol.cpp +++ b/reflector/M17Protocol.cpp @@ -54,8 +54,8 @@ void CM17Protocol::Task(void) std::unique_ptr Frame; // handle incoming packets -#if DSTAR_IPV6==true -#if DSTAR_IPV4==true +#if M17_IPV6==true +#if M17_IPV4==true if ( ReceiveDS(Buffer, Ip, 20) ) #else if ( Receive6(Buffer, Ip, 20) ) @@ -81,8 +81,6 @@ void CM17Protocol::Task(void) // push the "first" packet OnDvFramePacketIn(Frame, &Ip); - // leave a space between this pair of frames - std::this_thread::sleep_for(std::chrono::milliseconds(16)); // push the "second" packet OnDvFramePacketIn(secondFrame, &Ip); // push two packet because we need a packet every 20 ms } diff --git a/reflector/Main.cpp b/reflector/Main.cpp index 2ebd59b..db9968d 100644 --- a/reflector/Main.cpp +++ b/reflector/Main.cpp @@ -28,7 +28,7 @@ SJsonKeys g_Keys; CReflector g_Reflector; CGateKeeper g_GateKeeper; CConfigure g_Configure; -CVersion g_Version(3,1,0); // The major byte should only change if the interlink packet changes! +CVersion g_Version(3,1,1); // The major byte should only change if the interlink packet changes! CLookupDmr g_LDid; CLookupNxdn g_LNid; CLookupYsf g_LYtr; From 72a0a337f638afa9577a5c6efe5e2edcfcf8d2cb Mon Sep 17 00:00:00 2001 From: mfiscus Date: Sun, 26 Nov 2023 13:50:16 -0700 Subject: [PATCH 07/41] json api Ported (go/gin) json api from gomrefdash (https://github.com/kc1awv/gomrefdash) to (php/apache) urfd dashboard. This provides consistency between mrefd and urfd reflector dashboards and is intended to accelerate integration with dvref.com for an official URFD reflector registry. Updated dashboard version to 2.5.0 to help reflector admins determine compatibility with eventual dvref.com integration (>= 2.5.0). It has been extensively tested and is currently running on https://urf847.kk7mnz.com/json/reflector dvref.com integration has been tested at https://dvref.com/mrefd/m17-847 --- dashboard/json/.htaccess | 6 +++ dashboard/json/index.php | 63 ++++++++++++++++++++++++ dashboard/json/views/links.php | 24 +++++++++ dashboard/json/views/metadata.php | 23 +++++++++ dashboard/json/views/modulesinuse.php | 30 ++++++++++++ dashboard/json/views/peers.php | 22 +++++++++ dashboard/json/views/reflector.php | 70 +++++++++++++++++++++++++++ dashboard/json/views/stations.php | 36 ++++++++++++++ dashboard/json/views/status.php | 18 +++++++ dashboard/pgs/config.inc.php | 2 +- 10 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 dashboard/json/.htaccess create mode 100644 dashboard/json/index.php create mode 100644 dashboard/json/views/links.php create mode 100644 dashboard/json/views/metadata.php create mode 100644 dashboard/json/views/modulesinuse.php create mode 100644 dashboard/json/views/peers.php create mode 100644 dashboard/json/views/reflector.php create mode 100644 dashboard/json/views/stations.php create mode 100644 dashboard/json/views/status.php diff --git a/dashboard/json/.htaccess b/dashboard/json/.htaccess new file mode 100644 index 0000000..46623a5 --- /dev/null +++ b/dashboard/json/.htaccess @@ -0,0 +1,6 @@ +RewriteEngine On +RewriteBase /json/ +RewriteRule ^index\\.php$ - [L] +RewriteCond %{REQUEST_FILENAME} !-f +RewriteCond %{REQUEST_FILENAME} !-d +RewriteRule . /json/index.php [L] \ No newline at end of file diff --git a/dashboard/json/index.php b/dashboard/json/index.php new file mode 100644 index 0000000..90872a5 --- /dev/null +++ b/dashboard/json/index.php @@ -0,0 +1,63 @@ +SetXMLFile($Service['XMLFile']); +$Reflector->SetPIDFile($Service['PIDFile']); +$Reflector->LoadXML(); +$Reflector->SetFlagFile($_SERVER['DOCUMENT_ROOT'] . "/pgs/country.csv"); +$Reflector->LoadFlags(); + +$Request = $_SERVER['REQUEST_URI']; +$ViewDir = '/views/'; + +switch ($Request) { + case '/json/links': + require __DIR__ . $ViewDir . 'links.php'; + break; + + case '/json/metadata': + require __DIR__ . $ViewDir . 'metadata.php'; + break; + + case '/json/modulesinuse': + require __DIR__ . $ViewDir . 'modulesinuse.php'; + break; + + case '/json/peers': + require __DIR__ . $ViewDir . 'peers.php'; + break; + + case '/json/reflector': + require __DIR__ . $ViewDir . 'reflector.php'; + break; + + case '/json/stations': + require __DIR__ . $ViewDir . 'stations.php'; + break; + + case '/json/status': + require __DIR__ . $ViewDir . 'status.php'; + break; + + default: + header('Content-Type: text/plain'); + http_response_code(404); + echo('404 page not found'); + +} + +?> \ No newline at end of file diff --git a/dashboard/json/views/links.php b/dashboard/json/views/links.php new file mode 100644 index 0000000..ef34d51 --- /dev/null +++ b/dashboard/json/views/links.php @@ -0,0 +1,24 @@ +NodeCount();$i++) { + + // craft payload array + $payload[$i] = array( + 'callsign' => $Reflector->Nodes[$i]->GetCallSign() . ' ' . $Reflector->Nodes[$i]->GetSuffix(), + 'ip' => $Reflector->Nodes[$i]->GetIP(), + 'linkedmodule' => $Reflector->Nodes[$i]->GetLinkedModule(), + 'protocol' => $Reflector->Nodes[$i]->GetProtocol(), + 'connecttime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetConnectTime()), + 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetLastHeardTime()) + ); + +} + + +// json encode payload array +$records = json_encode($payload); + +echo $records; + +?> \ No newline at end of file diff --git a/dashboard/json/views/metadata.php b/dashboard/json/views/metadata.php new file mode 100644 index 0000000..fad5ed8 --- /dev/null +++ b/dashboard/json/views/metadata.php @@ -0,0 +1,23 @@ + $PageOptions['DashboardVersion'], + 'ipV4' => @$Net4[0]['ip'], + 'ipV6' => @$Net6[0]['ipv6'], + 'reflector_callsign' => str_replace("XLX", "URF", $Reflector->GetReflectorName()), + 'reflector_version' => $Reflector->GetVersion(), + 'sysop_email' => $PageOptions['ContactEmail'] +); + + +// json encode payload array +$records = json_encode($payload); + +echo $records; + +?> \ No newline at end of file diff --git a/dashboard/json/views/modulesinuse.php b/dashboard/json/views/modulesinuse.php new file mode 100644 index 0000000..85ba9c4 --- /dev/null +++ b/dashboard/json/views/modulesinuse.php @@ -0,0 +1,30 @@ +GetModules(); +sort($Modules, SORT_STRING); + + +/*** Add modules to payload ***/ +for ($i=0;$i $Modules[$i] + ); + + $Users = $Reflector->GetNodesInModulesByID($Modules[$i]); + + for ($j=0;$jGetCallsignAndSuffixByID($Users[$j]); + + } + +} + + +// json encode payload array +$records = json_encode($payload); + +echo $records; + +?> \ No newline at end of file diff --git a/dashboard/json/views/peers.php b/dashboard/json/views/peers.php new file mode 100644 index 0000000..d427c93 --- /dev/null +++ b/dashboard/json/views/peers.php @@ -0,0 +1,22 @@ +PeerCount();$i++) { + + $payload[$i] = array( + 'callsign' => $Reflector->Peers[$i]->GetCallSign(), + 'ip' => $Reflector->Peers[$i]->GetIP(), + 'linkedmodule' => $Reflector->Peers[$i]->GetLinkedModule(), + 'connecttime' => date('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetConnectTime()), + 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetLastHeardTime()) + ); + +} + + +// json encode payload array +$records = json_encode($payload); + +echo $records; + +?> \ No newline at end of file diff --git a/dashboard/json/views/reflector.php b/dashboard/json/views/reflector.php new file mode 100644 index 0000000..97c4b1f --- /dev/null +++ b/dashboard/json/views/reflector.php @@ -0,0 +1,70 @@ + date('Y-m-d\TH:i:sp', time()), + 'status' => $Status, + 'uptime' => $Reflector->GetServiceUptime() +); + + +/*** add data to payload ***/ +$payload['data'] = array( + 'filetime' => date('Y-m-d\TH:i:sp', filemtime($Service['XMLFile'])), + 'callsign' => str_replace("XLX", "URF", $Reflector->GetReflectorName()), + 'version' => $Reflector->GetVersion() +); + + +/*** Add peers to payload ***/ +for ($i=0;$i<$Reflector->PeerCount();$i++) { + + $payload['data']['peers'][] = array( + 'callsign' => $Reflector->Peers[$i]->GetCallSign(), + 'ip' => $Reflector->Peers[$i]->GetIP(), + 'linkedmodule' => $Reflector->Peers[$i]->GetLinkedModule(), + 'connecttime' => date('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetConnectTime()), + 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetLastHeardTime()) + ); + +} + + +/*** Add nodes to payload ***/ +for ($i=0;$i<$Reflector->NodeCount();$i++) { + + // craft payload array + $payload['data']['nodes'][] = array( + 'callsign' => $Reflector->Nodes[$i]->GetCallSign() . ' ' . $Reflector->Nodes[$i]->GetSuffix(), + 'ip' => $Reflector->Nodes[$i]->GetIP(), + 'linkedmodule' => $Reflector->Nodes[$i]->GetLinkedModule(), + 'protocol' => $Reflector->Nodes[$i]->GetProtocol(), + 'connecttime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetConnectTime()), + 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetLastHeardTime()) + ); + +} + + +/*** Add stations to payload ***/ +for ($i=0;$i<$Reflector->StationCount();$i++) { + + // craft payload array + $payload['data']['stations'][] = array( + 'callsign' => $Reflector->Stations[$i]->GetCallSign(), + 'vianode' => $Reflector->Stations[$i]->GetVia(), + 'onmodule' => $Reflector->Stations[$i]->GetModule(), + 'viapeer' => $Reflector->Stations[$i]->GetPeer(), + 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Stations[$i]->GetLastHeardTime()) + ); + +} + + +// json encode payload array +$records = json_encode($payload); + +echo $records; + +?> \ No newline at end of file diff --git a/dashboard/json/views/stations.php b/dashboard/json/views/stations.php new file mode 100644 index 0000000..fac3dae --- /dev/null +++ b/dashboard/json/views/stations.php @@ -0,0 +1,36 @@ +StationCount();$i++) { + + $tmp = preg_split('/\s+/', $Reflector->Stations[$i]->GetCallSign(), -1, PREG_SPLIT_NO_EMPTY); + $Callsign = $tmp[0]; + + $tmp = preg_split('/\s+/', $Reflector->Stations[$i]->GetVia(), -1, PREG_SPLIT_NO_EMPTY); + $CallsignSuffix = $tmp[1]; + + // craft payload array + $payload['stations'][$i] = array( + 'callsign' => $Callsign, + 'callsignsuffix' => $CallsignSuffix, + 'vianode' => $Reflector->Stations[$i]->GetVia(), + 'onmodule' => $Reflector->Stations[$i]->GetModule(), + 'lastheard' => date('Y-m-d\TH:i:sp', $Reflector->Stations[$i]->GetLastHeardTime()) + ); + + list ($CountryCode, $Country) = $Reflector->GetFlag($Reflector->Stations[$i]->GetCallSign()); + + $payload['stations'][$i]['country'] = array ( + 'country' => $Country, + 'countrycode' => $CountryCode + ); + +} + + +// json encode payload array +$records = json_encode($payload); + +echo $records; + +?> \ No newline at end of file diff --git a/dashboard/json/views/status.php b/dashboard/json/views/status.php new file mode 100644 index 0000000..947b450 --- /dev/null +++ b/dashboard/json/views/status.php @@ -0,0 +1,18 @@ + date('U', time()), + 'lasturfdupdate' => date('U', filemtime($Service['XMLFile'])), + 'reflectorstatus' => $ReflectorStatus, + 'reflectoruptimeseconds' => $Reflector->GetServiceUptime() +); + + +// json encode payload array +$records = json_encode($payload); + +echo $records; + +?> diff --git a/dashboard/pgs/config.inc.php b/dashboard/pgs/config.inc.php index ee027e0..bfd71d6 100644 --- a/dashboard/pgs/config.inc.php +++ b/dashboard/pgs/config.inc.php @@ -16,7 +16,7 @@ $PageOptions = array(); $PageOptions['ContactEmail'] = 'your_email'; // Support E-Mail address -$PageOptions['DashboardVersion'] = '2.4.1'; // Dashboard Version +$PageOptions['DashboardVersion'] = '2.5.0'; // Dashboard Version $PageOptions['PageRefreshActive'] = true; // Activate automatic refresh $PageOptions['PageRefreshDelay'] = '10000'; // Page refresh time in miliseconds From bdadea3cd1f9ede487d98e770e9c8f059cc0d78c Mon Sep 17 00:00:00 2001 From: Matt Fiscus Date: Tue, 28 Nov 2023 17:32:08 -0700 Subject: [PATCH 08/41] Fixed timezone issue Found an issue where timezone could be incorrect in api if server admin changes server timezeone to localtime instead of UTC. --- dashboard/json/index.php | 2 -- dashboard/json/views/links.php | 4 ++-- dashboard/json/views/peers.php | 4 ++-- dashboard/json/views/reflector.php | 6 +++--- dashboard/json/views/stations.php | 2 +- dashboard/json/views/status.php | 4 ++-- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/dashboard/json/index.php b/dashboard/json/index.php index 90872a5..7bcacad 100644 --- a/dashboard/json/index.php +++ b/dashboard/json/index.php @@ -3,8 +3,6 @@ header("Access-Control-Allow-Origin: *"); header('Content-Type: application/json'); -date_default_timezone_set("UTC"); - if (file_exists($_SERVER['DOCUMENT_ROOT'] . "/pgs/functions.php")) require_once($_SERVER['DOCUMENT_ROOT'] . "/pgs/functions.php"); if (file_exists($_SERVER['DOCUMENT_ROOT'] . "/pgs/config.inc.php")) require_once($_SERVER['DOCUMENT_ROOT'] . "/pgs/config.inc.php"); diff --git a/dashboard/json/views/links.php b/dashboard/json/views/links.php index ef34d51..6718f2d 100644 --- a/dashboard/json/views/links.php +++ b/dashboard/json/views/links.php @@ -9,8 +9,8 @@ for ($i=0;$i<$Reflector->NodeCount();$i++) { 'ip' => $Reflector->Nodes[$i]->GetIP(), 'linkedmodule' => $Reflector->Nodes[$i]->GetLinkedModule(), 'protocol' => $Reflector->Nodes[$i]->GetProtocol(), - 'connecttime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetConnectTime()), - 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetLastHeardTime()) + 'connecttime' => gmdate('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetConnectTime()), + 'lastheardtime' => gmdate('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetLastHeardTime()) ); } diff --git a/dashboard/json/views/peers.php b/dashboard/json/views/peers.php index d427c93..c3dc079 100644 --- a/dashboard/json/views/peers.php +++ b/dashboard/json/views/peers.php @@ -7,8 +7,8 @@ for ($i=0;$i<$Reflector->PeerCount();$i++) { 'callsign' => $Reflector->Peers[$i]->GetCallSign(), 'ip' => $Reflector->Peers[$i]->GetIP(), 'linkedmodule' => $Reflector->Peers[$i]->GetLinkedModule(), - 'connecttime' => date('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetConnectTime()), - 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetLastHeardTime()) + 'connecttime' => gmdate('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetConnectTime()), + 'lastheardtime' => gmdate('Y-m-d\TH:i:sp', $Reflector->Peers[$i]->GetLastHeardTime()) ); } diff --git a/dashboard/json/views/reflector.php b/dashboard/json/views/reflector.php index 97c4b1f..fe25d61 100644 --- a/dashboard/json/views/reflector.php +++ b/dashboard/json/views/reflector.php @@ -40,8 +40,8 @@ for ($i=0;$i<$Reflector->NodeCount();$i++) { 'ip' => $Reflector->Nodes[$i]->GetIP(), 'linkedmodule' => $Reflector->Nodes[$i]->GetLinkedModule(), 'protocol' => $Reflector->Nodes[$i]->GetProtocol(), - 'connecttime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetConnectTime()), - 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetLastHeardTime()) + 'connecttime' => gmdate('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetConnectTime()), + 'lastheardtime' => gmdate('Y-m-d\TH:i:sp', $Reflector->Nodes[$i]->GetLastHeardTime()) ); } @@ -56,7 +56,7 @@ for ($i=0;$i<$Reflector->StationCount();$i++) { 'vianode' => $Reflector->Stations[$i]->GetVia(), 'onmodule' => $Reflector->Stations[$i]->GetModule(), 'viapeer' => $Reflector->Stations[$i]->GetPeer(), - 'lastheardtime' => date('Y-m-d\TH:i:sp', $Reflector->Stations[$i]->GetLastHeardTime()) + 'lastheardtime' => gmdate('Y-m-d\TH:i:sp', $Reflector->Stations[$i]->GetLastHeardTime()) ); } diff --git a/dashboard/json/views/stations.php b/dashboard/json/views/stations.php index fac3dae..e0ae95e 100644 --- a/dashboard/json/views/stations.php +++ b/dashboard/json/views/stations.php @@ -15,7 +15,7 @@ for ($i=0;$i<$Reflector->StationCount();$i++) { 'callsignsuffix' => $CallsignSuffix, 'vianode' => $Reflector->Stations[$i]->GetVia(), 'onmodule' => $Reflector->Stations[$i]->GetModule(), - 'lastheard' => date('Y-m-d\TH:i:sp', $Reflector->Stations[$i]->GetLastHeardTime()) + 'lastheard' => gmdate('Y-m-d\TH:i:sp', $Reflector->Stations[$i]->GetLastHeardTime()) ); list ($CountryCode, $Country) = $Reflector->GetFlag($Reflector->Stations[$i]->GetCallSign()); diff --git a/dashboard/json/views/status.php b/dashboard/json/views/status.php index 947b450..5ebcb38 100644 --- a/dashboard/json/views/status.php +++ b/dashboard/json/views/status.php @@ -3,8 +3,8 @@ $ReflectorStatus = (file_exists($Service['PIDFile'])) ? 'up' : 'down'; $payload = array( - 'lastupdate' => date('U', time()), - 'lasturfdupdate' => date('U', filemtime($Service['XMLFile'])), + 'lastupdate' => gmdate('U', time()), + 'lasturfdupdate' => gmdate('U', filemtime($Service['XMLFile'])), 'reflectorstatus' => $ReflectorStatus, 'reflectoruptimeseconds' => $Reflector->GetServiceUptime() ); From f68754f0948197a213b90f1d0f30a754e210c8e1 Mon Sep 17 00:00:00 2001 From: coolacid Date: Wed, 13 Dec 2023 08:08:36 -0500 Subject: [PATCH 09/41] Correctly sets size in bind/connect to fixed name of abstract --- reflector/UnixDgramSocket.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/reflector/UnixDgramSocket.cpp b/reflector/UnixDgramSocket.cpp index ff21399..2075fdb 100644 --- a/reflector/UnixDgramSocket.cpp +++ b/reflector/UnixDgramSocket.cpp @@ -48,7 +48,9 @@ bool CUnixDgramReader::Open(const char *path) // returns true on failure addr.sun_family = AF_UNIX; strncpy(addr.sun_path+1, path, sizeof(addr.sun_path)-2); - int rval = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); + // We know path is a string, so we skip the first null, get the string length and add 1 for the begining Null + int path_len = sizeof(addr.sun_family) + strlen(addr.sun_path + 1) + 1; + int rval = bind(fd, (struct sockaddr *)&addr, path_len); if (rval < 0) { std::cerr << "bind() failed for " << path << ": " << strerror(errno) << std::endl; @@ -139,7 +141,9 @@ ssize_t CUnixDgramWriter::Write(const void *buf, ssize_t size) const return -1; } // connect to the receiver - int rval = connect(fd, (struct sockaddr *)&addr, sizeof(addr)); + // We know path is a string, so we skip the first null, get the string length and add 1 for the begining Null + int path_len = sizeof(addr.sun_family) + strlen(addr.sun_path + 1) + 1; + int rval = connect(fd, (struct sockaddr *)&addr, path_len); if (rval < 0) { std::cerr << "connect() failed for " << addr.sun_path+1 << ": " << strerror(errno) << std::endl; From 4d6928c6472652eb514c70417ac79f6755f77ddc Mon Sep 17 00:00:00 2001 From: Tom Early Date: Sun, 14 Apr 2024 15:14:10 -0700 Subject: [PATCH 10/41] fix for CCodecStream mismatch, fixed class.reflector.php, DHT Users and Clients are not permanent, fix CDvHeaderPacket DStar ctor --- dashboard/pgs/class.reflector.php | 138 +++++++++++++++--------------- reflector/CodecStream.cpp | 44 ++++++---- reflector/DVHeaderPacket.cpp | 11 --- reflector/Main.cpp | 2 +- reflector/Reflector.cpp | 4 +- 5 files changed, 95 insertions(+), 104 deletions(-) diff --git a/dashboard/pgs/class.reflector.php b/dashboard/pgs/class.reflector.php index d066819..e619979 100644 --- a/dashboard/pgs/class.reflector.php +++ b/dashboard/pgs/class.reflector.php @@ -1,7 +1,7 @@ Nodes = array(); $this->Stations = array(); @@ -35,61 +35,57 @@ class xReflector { $this->Interlinks = array(); $this->Transferinterlink = false; } - + public function LoadXML() { if ($this->XMLFile != null) { $handle = fopen($this->XMLFile, 'r'); $this->XMLContent = fread($handle, filesize($this->XMLFile)); fclose($handle); - + $this->ServiceName = substr($this->XMLContent, strpos($this->XMLContent, "ServiceName)) { - $this->ServiceName = null; - return false; - } - + $this->ReflectorName = "XLX".$this->ServiceName; - + $LinkedPeersName = "XLX".$this->ServiceName." linked peers"; $LinkedNodesName = "XLX".$this->ServiceName." linked nodes"; $LinkedUsersName = "XLX".$this->ServiceName." heard users"; - + $XML = new ParseXML(); - + $AllNodesString = $XML->GetElement($this->XMLContent, $LinkedNodesName); $tmpNodes = $XML->GetAllElements($AllNodesString, "NODE"); - + for ($i=0;$iGetElement($tmpNodes[$i], 'Callsign'), $XML->GetElement($tmpNodes[$i], 'IP'), $XML->GetElement($tmpNodes[$i], 'LinkedModule'), $XML->GetElement($tmpNodes[$i], 'Protocol'), $XML->GetElement($tmpNodes[$i], 'ConnectTime'), $XML->GetElement($tmpNodes[$i], 'LastHeardTime'), CreateCode(16)); $this->AddNode($Node); } - + $AllStationsString = $XML->GetElement($this->XMLContent, $LinkedUsersName); $tmpStations = $XML->GetAllElements($AllStationsString, "STATION"); for ($i=0;$iGetElement($tmpStations[$i], 'Callsign'), $XML->GetElement($tmpStations[$i], 'Via node'), $XML->GetElement($tmpStations[$i], 'Via peer'), $XML->GetElement($tmpStations[$i], 'LastHeardTime'), $XML->GetElement($tmpStations[$i], 'On module')); $this->AddStation($Station, false); } - + $AllPeersString = $XML->GetElement($this->XMLContent, $LinkedPeersName); $tmpPeers = $XML->GetAllElements($AllPeersString, "PEER"); for ($i=0;$iGetElement($tmpPeers[$i], 'Callsign'), $XML->GetElement($tmpPeers[$i], 'IP'), $XML->GetElement($tmpPeers[$i], 'LinkedModule'), $XML->GetElement($tmpPeers[$i], 'Protocol'), $XML->GetElement($tmpPeers[$i], 'ConnectTime'), $XML->GetElement($tmpPeers[$i], 'LastHeardTime')); $this->AddPeer($Peer, false); } - - $this->Version = $XML->GetElement($this->XMLContent, "Version"); + + $this->Version = $XML->GetElement($this->XMLContent, "Version"); } } - + public function GetVersion() { return $this->Version; } - + public function GetReflectorName() { return $this->ReflectorName; } - + public function SetXMLFile($XMLFile) { if (file_exists($XMLFile) && (is_readable($XMLFile))) { $this->XMLFile = $XMLFile; @@ -99,7 +95,7 @@ class xReflector { $this->XMLContent = null; } } - + public function SetPIDFile($ProcessIDFile) { if (file_exists($ProcessIDFile)) { $this->ProcessIDFile = $ProcessIDFile; @@ -110,11 +106,11 @@ class xReflector { $this->ServiceUptime = null; } } - + public function GetServiceUptime() { return $this->ServiceUptime; } - + public function SetFlagFile($Flagfile) { if (file_exists($Flagfile) && (is_readable($Flagfile))) { $this->Flagfile = $Flagfile; @@ -122,7 +118,7 @@ class xReflector { } return false; } - + public function LoadFlags() { if ($this->Flagfile != null) { $this->Flagarray = array(); @@ -133,18 +129,18 @@ class xReflector { while(!feof($handle)) { $row = fgets($handle,1024); $tmp = explode(";", $row); - + if (isset($tmp[0])) { $this->Flagarray[$i]['Country'] = $tmp[0]; } else { $this->Flagarray[$i]['Country'] = 'Undefined'; } if (isset($tmp[1])) { $this->Flagarray[$i]['ISO'] = $tmp[1]; } else { $this->Flagarray[$i]['ISO'] = "Undefined"; } //$this->Flagarray[$i]['DXCC'] = array(); - if (isset($tmp[2])) { + if (isset($tmp[2])) { $tmp2 = explode("-", $tmp[2]); for ($j=0;$jFlagarray[$i]['DXCC'][] = $tmp2[$j]; $this->Flagarray_DXCC[ trim($tmp2[$j]) ] = $i; } } - $i++; + $i++; } fclose($handle); } @@ -152,17 +148,17 @@ class xReflector { } return false; } - + public function AddNode($NodeObject) { if (is_object($NodeObject)) { $this->Nodes[] = $NodeObject; } } - + public function NodeCount() { return count($this->Nodes); } - + public function GetNode($ArrayIndex) { if (isset($this->Nodes[$ArrayIndex])) { return $this->Nodes[$ArrayIndex]; @@ -175,11 +171,11 @@ class xReflector { $this->Peers[] = $PeerObject; } } - + public function PeerCount() { return count($this->Peers); } - + public function GetPeer($ArrayIndex) { if (isset($this->Peer[$ArrayIndex])) { return $this->Peer[$ArrayIndex]; @@ -189,34 +185,34 @@ class xReflector { public function AddStation($StationObject, $AllowDouble = false) { if (is_object($StationObject)) { - + if ($AllowDouble) { $this->Stations[] = $StationObject; } else { $FoundStationInList = false; $i = 0; - + $tmp = explode(" ", $StationObject->GetCallsign()); $RealCallsign = trim($tmp[0]); - + while (!$FoundStationInList && $i<$this->StationCount()) { if ($this->Stations[$i]->GetCallsignOnly() == $RealCallsign) { $FoundStationInList = true; } $i++; } - + if (!$FoundStationInList) { if (strlen(trim($RealCallsign)) > 3) { $this->Stations[] = $StationObject; } } - + } } } - + public function GetSuffixOfRepeater($Repeater, $LinkedModul, $StartWithIndex = 0) { $suffix = ""; $found = false; @@ -232,7 +228,7 @@ class xReflector { } return $suffix; } - + public function GetCallsignAndSuffixByID($RandomId) { $suffix = ""; $callsign = ""; @@ -250,18 +246,18 @@ class xReflector { } return 'N/A'; } - + public function StationCount() { return count($this->Stations); } - + public function GetStation($ArrayIndex) { if (isset($this->Stations[$ArrayIndex])) { return $this->Stations[$ArrayIndex]; } return false; } - + public function GetFlag($Callsign) { $Image = ""; $Letters = 4; @@ -277,7 +273,7 @@ class xReflector { } return array(strtolower($Image), $Name); } - + public function GetModules() { $out = array(); for ($i=0;$i<$this->NodeCount();$i++) { @@ -295,7 +291,7 @@ class xReflector { } return $out; } - + public function GetModuleOfNode($Node) { die("FUNCTION DEPRECATED..."); $Node = trim(str_replace(" ", "-", $Node)); @@ -305,38 +301,38 @@ class xReflector { $i = 0; $Module = ""; while (!$found && $i<$this->NodeCount()) { - if (strpos($Node, $this->Nodes[$i]->GetFullCallsign()) !== false) { - $Module = $this->Nodes[$i]->GetLinkedModule(); + if (strpos($Node, $this->Nodes[$i]->GetFullCallsign()) !== false) { + $Module = $this->Nodes[$i]->GetLinkedModule(); $found = true; } - + $i++; } return $Module; - } - + } + public function GetCallSignsInModules($Module) { $out = array(); for ($i=0;$i<$this->NodeCount();$i++) { if ($this->Nodes[$i]->GetLinkedModule() == $Module) { $out[] = $this->Nodes[$i]->GetCallsign(); - } + } } return $out; } - + public function GetNodesInModulesById($Module) { $out = array(); for ($i=0;$i<$this->NodeCount();$i++) { if ($this->Nodes[$i]->GetLinkedModule() == $Module) { $out[] = $this->Nodes[$i]->GetRandomID(); - } + } } return $out; } - + public function SetCallingHome($CallingHomeVariables, $Hash) { - + if (!isset($CallingHomeVariables['Active'])) { $CallingHomeVariables['Active'] = false; } if (!isset($CallingHomeVariables['MyDashBoardURL'])) { $CallingHomeVariables['MyDashBoardURL'] = ''; } if (!isset($CallingHomeVariables['ServerURL'])) { $CallingHomeVariables['ServerURL'] = ''; } @@ -344,16 +340,16 @@ class xReflector { if (!isset($CallingHomeVariables['Comment'])) { $CallingHomeVariables['Comment'] = ''; } if (!isset($CallingHomeVariables['OverrideIPAddress'])) { $CallingHomeVariables['OverrideIPAddress'] = false; } if (!isset($CallingHomeVariables['InterlinkFile'])) { $CallingHomeVariables['InterlinkFile'] = ''; } - - if (!file_exists($CallingHomeVariables['InterlinkFile'])) { - $this->Interlinkfile = ''; + + if (!file_exists($CallingHomeVariables['InterlinkFile'])) { + $this->Interlinkfile = ''; $this->Transferinterlink = false; } else { $this->Transferinterlink = true; $this->Interlinkfile = $CallingHomeVariables['InterlinkFile']; } - + $this->CallingHomeActive = ($CallingHomeVariables['Active'] === true); $this->CallingHomeHash = $Hash; $this->CallingHomeDashboardURL = $CallingHomeVariables['MyDashBoardURL']; @@ -361,13 +357,13 @@ class xReflector { $this->CallingHomeCountry = $CallingHomeVariables['Country']; $this->CallingHomeComment = $CallingHomeVariables['Comment']; $this->CallingHomeOverrideIP = $CallingHomeVariables['OverrideIPAddress']; - + } - + public function PushCallingHome() { $CallingHome = @fopen($this->CallingHomeServerURL."?ReflectorName=".$this->ReflectorName."&ReflectorUptime=".$this->ServiceUptime."&ReflectorHash=".$this->CallingHomeHash."&DashboardURL=".$this->CallingHomeDashboardURL."&Country=".urlencode($this->CallingHomeCountry)."&Comment=".urlencode($this->CallingHomeComment)."&OverrideIP=".$this->CallingHomeOverrideIP, "r"); - } - + } + public function ReadInterlinkFile() { if (file_exists($this->Interlinkfile) && (is_readable($this->Interlinkfile))) { $this->Interlinks = array(); @@ -379,7 +375,7 @@ class xReflector { $this->Interlinks[] = new Interlink(); if (isset($Interlink[0])) { $this->Interlinks[count($this->Interlinks)-1]->SetName(trim($Interlink[0])); } if (isset($Interlink[1])) { $this->Interlinks[count($this->Interlinks)-1]->SetAddress(trim($Interlink[1])); } - if (isset($Interlink[2])) { + if (isset($Interlink[2])) { $Modules = str_split(trim($Interlink[2]), 1); for ($j=0;$jInterlinks[count($this->Interlinks)-1]->AddModule($Modules[$j]); @@ -391,7 +387,7 @@ class xReflector { } return false; } - + public function PrepareInterlinkXML() { $xml = ' '; @@ -407,7 +403,7 @@ class xReflector { '; $this->InterlinkXML = $xml; } - + public function PrepareReflectorXML() { $this->ReflectorXML = ' @@ -421,28 +417,28 @@ class xReflector { '.$this->Version.' '; } - + public function CallHome() { $xml = ' CallingHome'.$this->ReflectorXML.$this->InterlinkXML; $p = @stream_context_create(array('http' => array('header' => "Content-type: application/x-www-form-urlencoded\r\n", - 'method' => 'POST', + 'method' => 'POST', 'content' => http_build_query(array('xml' => $xml)) ))); $result = @file_get_contents($this->CallingHomeServerURL, false, $p); if ($result === false) { die("CONNECTION FAILED!"); } } - + public function InterlinkCount() { return count($this->Interlinks); } - + public function GetInterlink($Index) { if (isset($this->Interlinks[$Index])) return $this->Interlinks[$Index]; return array(); } - + public function IsInterlinked($Reflectorname) { $i = -1; $f = false; @@ -457,7 +453,7 @@ class xReflector { } return -1; } - + } ?> diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index afd7334..3bd1089 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -143,30 +143,36 @@ void CCodecStream::Task(void) { // pop the original packet auto Packet = m_LocalQueue.Pop(); - auto Frame = (CDvFramePacket *)Packet.get(); - - // do things look okay? - if (pack.module != m_CSModule) - std::cerr << "CodecStream '" << m_CSModule << "' received a transcoded packet from module '" << pack.module << "'" << std::dec << std::noshowbase << std::endl; - if (pack.sequence != Frame->GetCodecPacket()->sequence) - std::cerr << "Sequence mismatch: this voice frame=" << Frame->GetCodecPacket()->sequence << " returned transcoder packet=" << pack.sequence << std::endl; - if (pack.streamid != Frame->GetCodecPacket()->streamid) - std::cerr << std::hex << std::showbase << "StreamID mismatch: this voice frame=" << ntohs(Frame->GetCodecPacket()->streamid) << " returned transcoder packet=" << ntohs(pack.streamid) << std::dec << std::noshowbase << std::endl; - - // update content with transcoded data - Frame->SetCodecData(&pack); - // mark the DStar sync frames if the source isn't dstar - if (ECodecType::dstar!=Frame->GetCodecIn() && 0==Frame->GetPacketId()%21) + auto Frame = static_cast(Packet.get()); + + // make sure this is the correct packet + if ((pack.streamid == Frame->GetCodecPacket()->streamid) && (pack.sequence == Frame->GetCodecPacket()->sequence)) { - const uint8_t DStarSync[] = { 0x55, 0x2D, 0x16 }; - Frame->SetDvData(DStarSync); + // update content with transcoded data + Frame->SetCodecData(&pack); + // mark the DStar sync frames if the source isn't dstar + if (ECodecType::dstar!=Frame->GetCodecIn() && 0==Frame->GetPacketId()%21) + { + const uint8_t DStarSync[] = { 0x55, 0x2D, 0x16 }; + Frame->SetDvData(DStarSync); + } + + // and push it back to client + m_PacketStream->ReturnPacket(std::move(Packet)); + } + else + { + // Not the correct packet! It will be ignored + // Report it + if (pack.streamid != Frame->GetCodecPacket()->streamid) + std::cerr << std::hex << std::showbase << "StreamID mismatch: this voice frame=" << ntohs(Frame->GetCodecPacket()->streamid) << " returned transcoder packet=" << ntohs(pack.streamid) << std::dec << std::noshowbase << std::endl; + if (pack.sequence != Frame->GetCodecPacket()->sequence) + std::cerr << "Sequence mismatch: this voice frame=" << Frame->GetCodecPacket()->sequence << " returned transcoder packet=" << pack.sequence << std::endl; } - - // and push it back to client - m_PacketStream->ReturnPacket(std::move(Packet)); } else { + // Likewise, this packet will be ignored std::cout << "Transcoder packet received but CodecStream[" << m_CSModule << "] is closed: Module='" << pack.module << "' StreamID=" << std::hex << std::showbase << ntohs(pack.streamid) << std::endl; } } diff --git a/reflector/DVHeaderPacket.cpp b/reflector/DVHeaderPacket.cpp index d037faa..3272afc 100644 --- a/reflector/DVHeaderPacket.cpp +++ b/reflector/DVHeaderPacket.cpp @@ -90,17 +90,6 @@ CDvHeaderPacket::CDvHeaderPacket(const struct dstar_header *buffer, uint16_t sid m_uiFlag2 = buffer->Flag2; m_uiFlag3 = buffer->Flag3; m_csUR.SetCallsign(buffer->UR, CALLSIGN_LEN); - - if((buffer->RPT1)[7] == 0x20){ - char rptr1[8]; - memcpy(rptr1, buffer->RPT1, 8); - rptr1[7] = DPLUS_DEFAULT_RPTR1_SUFFIX; - m_csRPT1.SetCallsign(rptr1, CALLSIGN_LEN); - } - else{ - m_csRPT1.SetCallsign(buffer->RPT1, CALLSIGN_LEN); - } - m_csRPT1.SetCallsign(buffer->RPT1, CALLSIGN_LEN); m_csRPT2.SetCallsign(buffer->RPT2, CALLSIGN_LEN); m_csMY.SetCallsign(buffer->MY, CALLSIGN_LEN); diff --git a/reflector/Main.cpp b/reflector/Main.cpp index db9968d..3d5e1a5 100644 --- a/reflector/Main.cpp +++ b/reflector/Main.cpp @@ -28,7 +28,7 @@ SJsonKeys g_Keys; CReflector g_Reflector; CGateKeeper g_GateKeeper; CConfigure g_Configure; -CVersion g_Version(3,1,1); // The major byte should only change if the interlink packet changes! +CVersion g_Version(3,1,2); // The major byte should only change if the interlink packet changes! CLookupDmr g_LDid; CLookupNxdn g_LNid; CLookupYsf g_LYtr; diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index b02f8f7..68df588 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -595,7 +595,7 @@ void CReflector::PutDHTClients() #else [](bool success){ if (! success) std::cout << "PutDHTClients() unsuccessful" << std::endl; }, #endif - true // permanent! + false // not permanent! ); } @@ -624,7 +624,7 @@ void CReflector::PutDHTUsers() #else [](bool success){ if (! success) std::cout << "PutDHTUsers() unsuccessful" << std::endl; }, #endif - true // permanent! + false // not permanent ); } From 94f4689ee1be1a5d1f5f15d0207f0cb95a78c181 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Sun, 14 Apr 2024 19:57:29 -0700 Subject: [PATCH 11/41] added network id --- dashboard/pgs/class.reflector.php | 4 ++++ reflector/Reflector.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dashboard/pgs/class.reflector.php b/dashboard/pgs/class.reflector.php index e619979..8d6027e 100644 --- a/dashboard/pgs/class.reflector.php +++ b/dashboard/pgs/class.reflector.php @@ -43,6 +43,10 @@ class xReflector { fclose($handle); $this->ServiceName = substr($this->XMLContent, strpos($this->XMLContent, "ServiceName) == 1) { + $this->ServiceName = null; + return false; + } $this->ReflectorName = "XLX".$this->ServiceName; diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index 68df588..c65aa70 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -60,7 +60,7 @@ bool CReflector::Start(void) #ifndef NO_DHT // start the dht instance refhash = dht::InfoHash::get(cs); - node.run(17171, dht::crypto::generateIdentity(cs), true); + node.run(17171, dht::crypto::generateIdentity(cs), true, 59973); node.bootstrap(g_Configure.GetString(g_Keys.names.bootstrap), "17171"); #endif From ad47092c4ab88c22e5d523c208f0dce3c5ce8497 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Sun, 21 Apr 2024 13:42:12 -0700 Subject: [PATCH 12/41] in CUnixDGramWriter, only calculate path_len once --- reflector/UnixDgramSocket.cpp | 2 +- reflector/UnixDgramSocket.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/reflector/UnixDgramSocket.cpp b/reflector/UnixDgramSocket.cpp index 2075fdb..279eb6c 100644 --- a/reflector/UnixDgramSocket.cpp +++ b/reflector/UnixDgramSocket.cpp @@ -119,6 +119,7 @@ void CUnixDgramWriter::SetUp(const char *path) // returns true on failure memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; strncpy(addr.sun_path+1, path, sizeof(addr.sun_path)-2); + path_len = sizeof(addr.sun_family) + strlen(addr.sun_path + 1) + 1; } bool CUnixDgramWriter::Send(const STCPacket *pack) const @@ -142,7 +143,6 @@ ssize_t CUnixDgramWriter::Write(const void *buf, ssize_t size) const } // connect to the receiver // We know path is a string, so we skip the first null, get the string length and add 1 for the begining Null - int path_len = sizeof(addr.sun_family) + strlen(addr.sun_path + 1) + 1; int rval = connect(fd, (struct sockaddr *)&addr, path_len); if (rval < 0) { diff --git a/reflector/UnixDgramSocket.h b/reflector/UnixDgramSocket.h index 1f044e7..c590703 100644 --- a/reflector/UnixDgramSocket.h +++ b/reflector/UnixDgramSocket.h @@ -47,4 +47,5 @@ private: ssize_t Write(const void *buf, ssize_t size) const; struct sockaddr_un addr; + int path_len; }; From a5495deb857e019051aadf719fcb5aba94d254ff Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 3 May 2024 13:54:28 -0700 Subject: [PATCH 13/41] a new TCP socket --- config/urfd.ini | 9 +- reflector/CodecStream.cpp | 30 +++-- reflector/CodecStream.h | 5 - reflector/Configure.cpp | 97 +++++++++------ reflector/Configure.h | 2 +- reflector/JsonKeys.h | 11 +- reflector/Reflector.cpp | 23 +++- reflector/Reflector.h | 4 + reflector/TCTCPSocket.cpp | 223 ++++++++++++++++++++++++++++++++++ reflector/TCTCPSocket.h | 60 +++++++++ reflector/UnixDgramSocket.cpp | 167 ------------------------- reflector/UnixDgramSocket.h | 51 -------- 12 files changed, 396 insertions(+), 286 deletions(-) create mode 100644 reflector/TCTCPSocket.cpp create mode 100644 reflector/TCTCPSocket.h delete mode 100644 reflector/UnixDgramSocket.cpp delete mode 100644 reflector/UnixDgramSocket.h diff --git a/config/urfd.ini b/config/urfd.ini index 0aee3ac..77148a1 100644 --- a/config/urfd.ini +++ b/config/urfd.ini @@ -33,12 +33,9 @@ IPv4Binding = 0.0.0.0 # define if you want to override what urfd finds using ipv6.icanhazip.com # IPv6External = f:e:d:c:b:a:9:0 -Transcoder = local # SORRY, but only local TC's are supported right now! - [Modules] # Modules = ABCDEFGHIJKLMNOPQRSTUVWXYZ Modules = ADMSZ -Transcoded = A # comment out if you don't have transcoding hardware # Create Descriptions as needed... DescriptionA = Transcoded DescriptionD = DMR Chat @@ -46,6 +43,12 @@ DescriptionM = M17 Chat DescriptionS = DStar Chat DescriptionZ = Temp Meeting +[Transcoder] +Port = 10100 # TCP listening port for connection(s), set to 0 if there is no transcoder, then other two values will be ignored +BindingAddress = 127.0.0.1 # or ::1, the IPv4 or IPv6 "loop-back" address for a local transcoder +# For a connection to a remote transcoder, usually use the "any" address: 0.0.0.0 or :: +Modules = A # Transcoded modules one or three modules, depending on the hardware + # Protocols [Brandmeister] Enable = false # Set to true if you've configured BM connections in your urfd.interlink file. diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index 3bd1089..4fe09a4 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -18,9 +18,14 @@ #include +#include + #include "DVFramePacket.h" #include "PacketStream.h" #include "CodecStream.h" +#include "Reflector.h" + +extern CReflector g_Reflector; //////////////////////////////////////////////////////////////////////////////////////// // constructor @@ -42,7 +47,6 @@ CCodecStream::~CCodecStream() m_Future.get(); } // and close the socket - m_TCReader.Close(); } void CCodecStream::ResetStats(uint16_t streamid, ECodecType type) @@ -80,12 +84,6 @@ void CCodecStream::ReportStats() bool CCodecStream::InitCodecStream() { - m_TCWriter.SetUp(REF2TC); - std::string name(TC2REF); - name.append(1, m_CSModule); - if (m_TCReader.Open(name.c_str())) - return true; - std::cout << "Initialized CodecStream receive socket " << name << std::endl; keep_running = true; try { @@ -94,7 +92,6 @@ bool CCodecStream::InitCodecStream() catch(const std::exception& e) { std::cerr << "Could not start Codec processing on module '" << m_CSModule << "': " << e.what() << std::endl; - m_TCReader.Close(); return true; } return false; @@ -114,10 +111,21 @@ void CCodecStream::Thread() void CCodecStream::Task(void) { STCPacket pack; + struct timeval tv; + fd_set readfds; + + tv.tv_sec = 0; + tv.tv_usec = 7000; + + int fd = g_Reflector.tcServer.GetFD(m_CSModule); + + FD_ZERO(&readfds); + FD_SET(fd, &readfds); - // any packet from transcoder - if (m_TCReader.Receive(&pack, 5)) + // don't care about writefds and exceptfds: + if (select(fd+1, &readfds, NULL, NULL, &tv)) { + g_Reflector.tcServer.Receive(fd, &pack); // update statistics double rt = pack.rt_timer.time(); // the round-trip time if (0 == m_RTCount) @@ -191,7 +199,7 @@ void CCodecStream::Task(void) Frame->SetTCParams(m_uiTotalPackets++); // now send to transcoder - m_TCWriter.Send(Frame->GetCodecPacket()); + g_Reflector.tcServer.Send(Frame->GetCodecPacket()); // push to our local queue where it can wait for the transcoder m_LocalQueue.Push(std::move(Packet)); diff --git a/reflector/CodecStream.h b/reflector/CodecStream.h index 1576f1d..f23081b 100644 --- a/reflector/CodecStream.h +++ b/reflector/CodecStream.h @@ -22,7 +22,6 @@ #include #include "DVFramePacket.h" -#include "UnixDgramSocket.h" #include "SafePacketQueue.h" //////////////////////////////////////////////////////////////////////////////////////// @@ -64,10 +63,6 @@ protected: uint8_t m_uiPid; ECodecType m_eCodecIn; - // sockets - CUnixDgramReader m_TCReader; - CUnixDgramWriter m_TCWriter; - // associated packet stream CPacketStream *m_PacketStream; diff --git a/reflector/Configure.cpp b/reflector/Configure.cpp index 21b837e..9abe487 100644 --- a/reflector/Configure.cpp +++ b/reflector/Configure.cpp @@ -32,6 +32,7 @@ // ini file keywords #define JAUTOLINKMODULE "AutoLinkModule" +#define JBINDINGADDRESS "BindingAddress" #define JBLACKLISTPATH "BlacklistPath" #define JBOOTSTRAP "Bootstrap" #define JBRANDMEISTER "Brandmeister" @@ -79,7 +80,6 @@ #define JRXPORT "RxPort" #define JSPONSOR "Sponsor" #define JSYSOPEMAIL "SysopEmail" -#define JTRANSCODED "Transcoded" #define JTRANSCODER "Transcoder" #define JTXPORT "TxPort" #define JURF "URF" @@ -131,6 +131,7 @@ bool CConfigure::ReadData(const std::string &path) ESection section = ESection::none; counter = 0; SJsonKeys::DB *pdb; + unsigned tcport = 0; //data.ysfalmodule = 0; //data.DPlusPort = data.DCSPort = data.DExtraPort = data.BMPort = data.DMRPlusPort = 0; @@ -180,6 +181,8 @@ bool CConfigure::ReadData(const std::string &path) section = ESection::names; else if (0 == hname.compare(JIPADDRESSES)) section = ESection::ip; + else if (0 == hname.compare(JTRANSCODER)) + section = ESection::tc; else if (0 == hname.compare(JMODULES)) section = ESection::modules; else if (0 == hname.compare(JDPLUS)) @@ -282,17 +285,26 @@ bool CConfigure::ReadData(const std::string &path) { data[g_Keys.ip.ipv6address] = value; } - else if (0 == key.compare(JTRANSCODER)) + else + badParam(key); + break; + case ESection::tc: + if (0 == key.compare(JPORT)) + data[g_Keys.tc.port] = getUnsigned(value, "Transcoder Port", 0, 40000, 10100); + else if (key.compare(JBINDINGADDRESS)) + data[g_Keys.tc.bind] = value; + else if (key.compare(JMODULES)) { - if (value.compare("local")) + std::string m(value); + if (checkModules(m)) { - std::cout << "WARNING: Line #" << counter << ": malformed transcoder address, '" << value << "', resetting..." << std::endl; + std::cerr << "ERROR: line #" << counter << ": no letters found in transcoder Modules: '" << m << "'" << std::endl; + rval = true; } - data[g_Keys.ip.transcoder] = "local"; + data[g_Keys.tc.modules] = m; } else badParam(key); - break; case ESection::modules: if (0 == key.compare(JMODULES)) { @@ -304,16 +316,6 @@ bool CConfigure::ReadData(const std::string &path) } else data[g_Keys.modules.modules] = m; } - else if (0 == key.compare(JTRANSCODED)) - { - std::string m(value); - if (checkModules(m)) - { - std::cerr << "ERROR: line #" << counter << ": no letters found in Transcoded: '" << m << "'" << std::endl; - rval = true; - } else - data[g_Keys.modules.tcmodules] = m; - } else if (0 == key.compare(0, 11, "Description")) { if (12 == key.size() && isupper(key[11])) @@ -606,27 +608,6 @@ bool CConfigure::ReadData(const std::string &path) if (isDefined(ErrorLevel::fatal, JMODULES, JMODULES, g_Keys.modules.modules, rval)) { const auto mods(data[g_Keys.modules.modules].get()); - if (data.contains(g_Keys.modules.tcmodules)) - { - const auto tcmods(data[g_Keys.modules.tcmodules].get()); - - // how many transcoded modules - auto size = tcmods.size(); - if (3 != size && 1 != size) - std::cout << "WARNING: [" << JMODULES << ']' << JTRANSCODED << " doesn't define one (or three) modules" << std::endl; - - // make sure each transcoded module is configured - for (auto c : tcmods) - { - if (std::string::npos == mods.find(c)) - { - std::cerr << "ERROR: transcoded module '" << c << "' not found in defined modules" << std::endl; - rval = true; - } - } - } - else - data[g_Keys.modules.tcmodules] = nullptr; // finally, check the module descriptions for (unsigned i=0; i<26; i++) @@ -652,6 +633,44 @@ bool CConfigure::ReadData(const std::string &path) } } + // Transcoder section + if (isDefined(ErrorLevel::fatal, JTRANSCODER, JPORT, g_Keys.tc.port, rval)) + { + tcport = GetUnsigned(g_Keys.tc.port); + if (tcport) + { + if (isDefined(ErrorLevel::fatal, JTRANSCODER, JBINDINGADDRESS, g_Keys.tc.bind, rval)) + { + const auto bind(data[g_Keys.tc.bind].get()); + if (!std::regex_match(bind, IPv4RegEx) && !std::regex_match(bind, IPv6RegEx)) + { + std::cerr << "ERROR: Transcoder bind address [" << bind << "] is malformed" << std::endl; + rval = true; + } + } + if (isDefined(ErrorLevel::fatal, JTRANSCODER, JMODULES, g_Keys.tc.modules, rval)) + { + const auto tcmods(data[g_Keys.tc.modules].get()); + + // how many transcoded modules + auto size = tcmods.size(); + if (3 != size && 1 != size) + std::cout << "WARNING: [" << JTRANSCODER << ']' << JMODULES << " doesn't define one (or three) modules" << std::endl; + + // make sure each transcoded module is configured + const std::string mods(GetString(g_Keys.modules.modules)); + for (auto c : tcmods) + { + if (std::string::npos == mods.find(c)) + { + std::cerr << "ERROR: transcoded module '" << c << "' not found in defined modules" << std::endl; + rval = true; + } + } + } + } + } + // "simple" protocols with only a Port isDefined(ErrorLevel::fatal, JDCS, JPORT, g_Keys.dcs.port, rval); isDefined(ErrorLevel::fatal, JDEXTRA, JPORT, g_Keys.dextra.port, rval); @@ -691,11 +710,11 @@ bool CConfigure::ReadData(const std::string &path) { if (GetBoolean(g_Keys.usrp.enable)) { - if (IsString(g_Keys.modules.tcmodules)) + if (tcport) { if (isDefined(ErrorLevel::fatal, JUSRP, JMODULE, g_Keys.usrp.module, rval)) { - if (std::string::npos == GetString(g_Keys.modules.tcmodules).find(GetString(g_Keys.usrp.module).at(0))) + if (std::string::npos == GetString(g_Keys.tc.modules).find(GetString(g_Keys.usrp.module).at(0))) { std::cerr << "ERROR: [" << JUSRP << ']' << JMODULE << " is not a transcoded module" << std::endl; rval = true; diff --git a/reflector/Configure.h b/reflector/Configure.h index 18e6161..fb27055 100644 --- a/reflector/Configure.h +++ b/reflector/Configure.h @@ -25,7 +25,7 @@ enum class ErrorLevel { fatal, mild }; enum class ERefreshType { file, http, both }; -enum class ESection { none, names, ip, modules, urf, dplus, dextra, dcs, g3, dmrplus, mmdvm, nxdn, bm, ysf, p25, m17, usrp, dmrid, nxdnid, ysffreq, files }; +enum class ESection { none, names, ip, modules, urf, dplus, dextra, dcs, g3, dmrplus, mmdvm, nxdn, bm, ysf, p25, m17, usrp, dmrid, nxdnid, ysffreq, files, tc }; #define IS_TRUE(a) ((a)=='t' || (a)=='T' || (a)=='1') diff --git a/reflector/JsonKeys.h b/reflector/JsonKeys.h index ab19407..72cfc0d 100644 --- a/reflector/JsonKeys.h +++ b/reflector/JsonKeys.h @@ -42,11 +42,14 @@ struct SJsonKeys { struct NAMES { const std::string callsign, bootstrap, url, email, country, sponsor; } names { "Callsign", "bootstrap", "DashboardUrl", "SysopEmail", "Country", "Sponsor" }; - struct IP { const std::string ipv4bind, ipv4address, ipv6bind, ipv6address, transcoder; } - ip { "ipv4bind", "IPv4Address", "ipv6bind", "IPv6Address", "tcaddress" }; + struct IP { const std::string ipv4bind, ipv4address, ipv6bind, ipv6address; } + ip { "ipv4bind", "IPv4Address", "ipv6bind", "IPv6Address" }; - struct MODULES { const std::string modules, tcmodules, descriptor[26]; } - modules { "Modules", "TranscodedModules", + struct TC { const std::string port, bind, modules; } + tc { "tcport", "tcbind", "TranscodedModules" }; + + struct MODULES { const std::string modules, descriptor[26]; } + modules { "Modules", "DescriptionA", "DescriptionB", "DescriptionC", "DescriptionD", "DescriptionE", "DescriptionF", "DescriptionG", "DescriptionH", "DescriptionI", "DescriptionJ", "DescriptionK", "DescriptionL", "DescriptionM", "DescriptionN", "DescriptionO", "DescriptionP", "DescriptionQ", "DescriptionR", "DescriptionS", "DescriptionT", "DescriptionU", "DescriptionV", "DescriptionW", "DescriptionX", "DescriptionY", "DescriptionZ" }; struct USRP { const std::string enable, ip, txport, rxport, module, callsign, filepath; } diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index c65aa70..08da97a 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -55,7 +55,8 @@ bool CReflector::Start(void) const auto cs(g_Configure.GetString(g_Keys.names.callsign)); m_Callsign.SetCallsign(cs, false); m_Modules.assign(g_Configure.GetString(g_Keys.modules.modules)); - std::string tcmods(g_Configure.GetString(g_Keys.modules.tcmodules)); + const auto tcmods(g_Configure.GetString(g_Keys.tc.modules)); + const auto port = g_Configure.GetUnsigned(g_Keys.tc.port); #ifndef NO_DHT // start the dht instance @@ -67,6 +68,10 @@ bool CReflector::Start(void) // let's go! keep_running = true; + // init transcoder comms + if (port) + tcServer.Open(g_Configure.GetString(g_Keys.tc.bind), tcmods, port); + // init gate keeper. It can only return true! g_GateKeeper.Init(); @@ -93,10 +98,13 @@ bool CReflector::Start(void) if (stream) { // if it's a transcoded module, then we need to initialize the codec stream - if (std::string::npos != tcmods.find(c)) + if (port) { - if (stream->InitCodecStream()) - return true; + if (std::string::npos != tcmods.find(c)) + { + if (stream->InitCodecStream()) + return true; + } } m_Stream[c] = stream; } @@ -139,6 +147,10 @@ void CReflector::Stop(void) // stop & delete all threads keep_running = false; + // stop transcoder comms + // if it was never opened, then there is nothing to close; + tcServer.Close(); + // stop & delete report threads if ( m_XmlReportFuture.valid() ) { @@ -637,7 +649,8 @@ void CReflector::PutDHTConfig() cfg.ipv4addr.assign(g_Configure.GetString(g_Keys.ip.ipv4address)); cfg.ipv6addr.assign(g_Configure.GetString(g_Keys.ip.ipv6address)); cfg.modules.assign(g_Configure.GetString(g_Keys.modules.modules)); - cfg.transcodedmods.assign(g_Configure.GetString(g_Keys.modules.tcmodules)); + if (g_Configure.GetUnsigned(g_Keys.tc.port)) + cfg.transcodedmods.assign(g_Configure.GetString(g_Keys.tc.modules)); cfg.url.assign(g_Configure.GetString(g_Keys.names.url)); cfg.email.assign(g_Configure.GetString(g_Keys.names.email)); cfg.country.assign(g_Configure.GetString(g_Keys.names.country)); diff --git a/reflector/Reflector.h b/reflector/Reflector.h index 5d8fb52..30f04d2 100644 --- a/reflector/Reflector.h +++ b/reflector/Reflector.h @@ -25,6 +25,7 @@ #include "Peers.h" #include "Protocols.h" #include "PacketStream.h" +#include "TCTCPSocket.h" #ifndef NO_DHT #include "dht-values.h" @@ -83,6 +84,9 @@ public: void GetDHTConfig(const std::string &cs); #endif + // transcoder communication + CTCTCPServer tcServer; + protected: #ifndef NO_DHT // Publish DHT diff --git a/reflector/TCTCPSocket.cpp b/reflector/TCTCPSocket.cpp new file mode 100644 index 0000000..3bbcabb --- /dev/null +++ b/reflector/TCTCPSocket.cpp @@ -0,0 +1,223 @@ +// urfd -- The universal reflector +// Copyright © 2024 Thomas A. Early N7TAE +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include +#include +#include +#include +#include +#include + +#include "IP.h" +#include "TCTCPSocket.h" + +void CTCTCPSocket::Close() +{ + for (auto item : m_FD) + close(item.second); + m_FD.clear(); +} + +void CTCTCPSocket::Close(char mod) +{ + auto item = m_FD.find(mod); + if (m_FD.end() == item) + return; + + close(item->second); + m_FD.erase(item); +} + +bool CTCTCPSocket::Send(const STCPacket *packet) +{ + int fd = GetFD(packet->module); + if (fd < 0) + { + return true; + } + + long count = 0; + auto data = (const unsigned char *)packet; + do { + auto n = send(fd, data+count, sizeof(STCPacket)-count, 0); + if (n <= 0) + { + if (0 == n) + { + std::cerr << "CTCTCPSocket::Send: socket on module '" << packet->module << "' has been closed!" << std::endl; + } + else + { + perror("CTCTCPSocket::Send"); + } + Close(packet->module); + return true; + } + count += n; + } while (count < sizeof(STCPacket)); + return false; +} + +bool CTCTCPSocket::Receive(int fd, STCPacket *packet) +{ + auto data = (unsigned char *)packet; + auto n = recv(fd, data, sizeof(STCPacket), MSG_WAITALL); + if (n < 0) + { + perror("CTCTCPSocket::Receive"); + return true; + } + return n == sizeof(STCPacket); +} + +int CTCTCPSocket::GetFD(char module) const +{ + const auto item = m_FD.find(module); + if (m_FD.cend() == item) + { + return -1; + } + return item->second; +} + +bool CTCTCPServer::Open(const std::string &address, const std::string &tcmodules, uint16_t port) +{ + int fd; + CIp ip(address.c_str(), AF_UNSPEC, SOCK_STREAM, port); + + fd = socket(ip.GetFamily(), SOCK_STREAM, 0); + if (fd < 0) + { + perror("Open socket"); + return true; + } + + int yes = 1; + int rv = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)); + if (rv < 0) + { + close(fd); + perror("Open setsockopt"); + return true; + } + + rv = bind(fd, ip.GetCPointer(), ip.GetSize()); + if (rv < 0) + { + close(fd); + perror("Open bind"); + return true; + } + + rv = listen(fd, 3); + if (rv < 0) + { + perror("Open listen"); + close(fd); + Close(); + return true; + } + + std::cout << "Waiting for " << tcmodules.size() << " transcoder connection(s)..." << std::endl; + for (unsigned x=0; x. + +#pragma once + +#include +#include +#include + +#include "TCPacketDef.h" + +class CTCTCPSocket +{ +public: + CTCTCPSocket() {} + virtual ~CTCTCPSocket() { Close(); } + + void Close(); // close all open sockets + + // bool functions return true on failure + virtual bool Open(const std::string &address, const std::string &modules, uint16_t port) = 0; + bool Send(const STCPacket *packet); + bool Receive(int fd, STCPacket *packet); + + int GetFD(char module) const; // can return -1! + +protected: + void Close(char); // close a specific module + + std::unordered_map m_FD; +}; + +class CTCTCPServer : public CTCTCPSocket +{ +public: + CTCTCPServer() : CTCTCPSocket() {} + ~CTCTCPServer() {} + bool Open(const std::string &address, const std::string &modules, uint16_t port); +}; + +class CTCTCPClient : public CTCTCPSocket +{ +public: + CTCTCPClient() : CTCTCPSocket() {} + ~CTCTCPClient() {} + bool Open(const std::string &address, const std::string &modules, uint16_t port); +}; diff --git a/reflector/UnixDgramSocket.cpp b/reflector/UnixDgramSocket.cpp deleted file mode 100644 index 279eb6c..0000000 --- a/reflector/UnixDgramSocket.cpp +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright © 2021 Thomas A. Early N7TAE -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "UnixDgramSocket.h" - -CUnixDgramReader::CUnixDgramReader() : fd(-1) {} - -CUnixDgramReader::~CUnixDgramReader() -{ - Close(); -} - -bool CUnixDgramReader::Open(const char *path) // returns true on failure -{ - fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if (fd < 0) - { - std::cerr << "socket() failed for " << path << ": " << strerror(errno) << std::endl; - return true; - } - //fcntl(fd, F_SETFL, O_NONBLOCK); - - struct sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strncpy(addr.sun_path+1, path, sizeof(addr.sun_path)-2); - - // We know path is a string, so we skip the first null, get the string length and add 1 for the begining Null - int path_len = sizeof(addr.sun_family) + strlen(addr.sun_path + 1) + 1; - int rval = bind(fd, (struct sockaddr *)&addr, path_len); - if (rval < 0) - { - std::cerr << "bind() failed for " << path << ": " << strerror(errno) << std::endl; - close(fd); - fd = -1; - return true; - } - return false; -} - -bool CUnixDgramReader::Receive(STCPacket *pack, unsigned timeout) const -{ - // socket valid ? - if ( 0 > fd ) - return false; - - // control socket - fd_set FdSet; - FD_ZERO(&FdSet); - FD_SET(fd, &FdSet); - struct timeval tv; - tv.tv_sec = timeout / 1000; - tv.tv_usec = (timeout % 1000) * 1000; - - auto rval = select(fd + 1, &FdSet, 0, 0, &tv); - if (rval <= 0) { - if (rval < 0) { - std::cerr << "select() error on transcoder socket: " << strerror(errno) << std::endl; - } - return false; - } - - return Read(pack); -} - -bool CUnixDgramReader::Read(STCPacket *pack) const -{ - auto len = read(fd, pack, sizeof(STCPacket)); - if (len != sizeof(STCPacket)) { - std::cerr << "Received transcoder packet is wrong size: " << len << " but should be " << sizeof(STCPacket) << std::endl; - return false; - } - - return true; -} - -void CUnixDgramReader::Close() -{ - if (fd >= 0) - close(fd); - fd = -1; -} - -int CUnixDgramReader::GetFD() const -{ - return fd; -} - -CUnixDgramWriter::CUnixDgramWriter() {} - -CUnixDgramWriter::~CUnixDgramWriter() {} - -void CUnixDgramWriter::SetUp(const char *path) // returns true on failure -{ - // setup the socket address - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strncpy(addr.sun_path+1, path, sizeof(addr.sun_path)-2); - path_len = sizeof(addr.sun_family) + strlen(addr.sun_path + 1) + 1; -} - -bool CUnixDgramWriter::Send(const STCPacket *pack) const -{ - auto len = Write(pack, sizeof(STCPacket)); - - if (len != sizeof(STCPacket)) - return true; - - return false; -} - -ssize_t CUnixDgramWriter::Write(const void *buf, ssize_t size) const -{ - // open the socket - int fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if (fd < 0) - { - std::cerr << "socket() failed for " << addr.sun_path+1 << ": " << strerror(errno) << std::endl; - return -1; - } - // connect to the receiver - // We know path is a string, so we skip the first null, get the string length and add 1 for the begining Null - int rval = connect(fd, (struct sockaddr *)&addr, path_len); - if (rval < 0) - { - std::cerr << "connect() failed for " << addr.sun_path+1 << ": " << strerror(errno) << std::endl; - close(fd); - return -1; - } - - auto written = write(fd, buf, size); - if (written != size) { - std::cerr << "write on " << addr.sun_path+1; - if (written < 0) - std::cerr << " returned error: " << strerror(errno) << std::endl; - else if (written == 0) - std::cerr << " returned zero" << std::endl; - else - std::cerr << " only wrote " << written << " bytes, should be " << size << std::endl; - } - - close(fd); - return written; -} diff --git a/reflector/UnixDgramSocket.h b/reflector/UnixDgramSocket.h deleted file mode 100644 index c590703..0000000 --- a/reflector/UnixDgramSocket.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -// Copyright © 2021 Thomas A. Early N7TAE -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#pragma once - -#include -#include - -#include "TCPacketDef.h" - -class CUnixDgramReader -{ -public: - CUnixDgramReader(); - ~CUnixDgramReader(); - bool Open(const char *path); - bool Read(STCPacket *pack) const; - bool Receive(STCPacket *pack, unsigned timeout) const; - void Close(); - int GetFD() const; -private: - int fd; -}; - -class CUnixDgramWriter -{ -public: - CUnixDgramWriter(); - ~CUnixDgramWriter(); - void SetUp(const char *path); - bool Send(const STCPacket *pack) const; -private: - ssize_t Write(const void *buf, ssize_t size) const; - - struct sockaddr_un addr; - int path_len; -}; From cb7bb2af3597e732390ed6d462240a52d57dd411 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Wed, 8 May 2024 12:31:05 -0700 Subject: [PATCH 14/41] it compiles with new TCP socket --- reflector/BMProtocol.cpp | 2 +- reflector/Buffer.h | 1 + reflector/CodecStream.cpp | 69 ++++-- reflector/CodecStream.h | 4 +- reflector/Configure.cpp | 7 + reflector/Global.h | 2 + reflector/Main.cpp | 1 + reflector/PacketStream.cpp | 6 +- reflector/Reflector.cpp | 4 +- reflector/Reflector.h | 4 - reflector/SafePacketQueue.h | 13 ++ reflector/TCSocket.cpp | 294 ++++++++++++++++++++++++ reflector/{TCTCPSocket.h => TCSocket.h} | 38 +-- reflector/TCTCPSocket.cpp | 223 ------------------ reflector/URFProtocol.cpp | 2 +- 15 files changed, 396 insertions(+), 274 deletions(-) create mode 100644 reflector/TCSocket.cpp rename reflector/{TCTCPSocket.h => TCSocket.h} (62%) delete mode 100644 reflector/TCTCPSocket.cpp diff --git a/reflector/BMProtocol.cpp b/reflector/BMProtocol.cpp index bc0663a..3f211b8 100644 --- a/reflector/BMProtocol.cpp +++ b/reflector/BMProtocol.cpp @@ -27,7 +27,7 @@ bool CBMProtocol::Initialize(const char *type, const EProtocol ptype, const uint16_t port, const bool has_ipv4, const bool has_ipv6) { - m_HasTranscoder = g_Configure.IsString(g_Keys.modules.tcmodules); + m_HasTranscoder = (0 != g_Configure.GetUnsigned(g_Keys.tc.port)); if (! CProtocol::Initialize(type, ptype, port, has_ipv4, has_ipv6)) return false; diff --git a/reflector/Buffer.h b/reflector/Buffer.h index 12eb495..c007ef8 100644 --- a/reflector/Buffer.h +++ b/reflector/Buffer.h @@ -20,6 +20,7 @@ #include #include +#include //////////////////////////////////////////////////////////////////////////////////////// diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index 4fe09a4..bde8508 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -20,13 +20,12 @@ #include #include +#include "Global.h" #include "DVFramePacket.h" #include "PacketStream.h" #include "CodecStream.h" #include "Reflector.h" -extern CReflector g_Reflector; - //////////////////////////////////////////////////////////////////////////////////////// // constructor @@ -110,6 +109,21 @@ void CCodecStream::Thread() void CCodecStream::Task(void) { + int fd = g_TCServer.GetFD(m_CSModule); + if (fd < 0) // log the situation + std::cout << "Lost connection to transcoder, module '" << m_CSModule << "', waiting..." << std::endl; + while (fd < 0) + { + if (g_TCServer.Accept()) // we will block until we have a new connection + { + std::cerr << "UNRECOVERABLE ERROR!" << std::endl; + exit(1); + } + // Accept was sucessful, but we need to make sure THIS MODULE was reestablished + // It's possile that other Transcoder ports were lost + fd = g_TCServer.GetFD(m_CSModule); + } + STCPacket pack; struct timeval tv; fd_set readfds; @@ -117,15 +131,14 @@ void CCodecStream::Task(void) tv.tv_sec = 0; tv.tv_usec = 7000; - int fd = g_Reflector.tcServer.GetFD(m_CSModule); - FD_ZERO(&readfds); FD_SET(fd, &readfds); // don't care about writefds and exceptfds: if (select(fd+1, &readfds, NULL, NULL, &tv)) { - g_Reflector.tcServer.Receive(fd, &pack); + if (g_TCServer.Receive(fd, &pack)) + return; // update statistics double rt = pack.rt_timer.time(); // the round-trip time if (0 == m_RTCount) @@ -151,18 +164,17 @@ void CCodecStream::Task(void) { // pop the original packet auto Packet = m_LocalQueue.Pop(); - auto Frame = static_cast(Packet.get()); // make sure this is the correct packet - if ((pack.streamid == Frame->GetCodecPacket()->streamid) && (pack.sequence == Frame->GetCodecPacket()->sequence)) + if ((pack.streamid == Packet->GetCodecPacket()->streamid) && (pack.sequence == Packet->GetCodecPacket()->sequence)) { // update content with transcoded data - Frame->SetCodecData(&pack); + Packet->SetCodecData(&pack); // mark the DStar sync frames if the source isn't dstar - if (ECodecType::dstar!=Frame->GetCodecIn() && 0==Frame->GetPacketId()%21) + if (ECodecType::dstar!=Packet->GetCodecIn() && 0==Packet->GetPacketId()%21) { const uint8_t DStarSync[] = { 0x55, 0x2D, 0x16 }; - Frame->SetDvData(DStarSync); + Packet->SetDvData(DStarSync); } // and push it back to client @@ -172,10 +184,10 @@ void CCodecStream::Task(void) { // Not the correct packet! It will be ignored // Report it - if (pack.streamid != Frame->GetCodecPacket()->streamid) - std::cerr << std::hex << std::showbase << "StreamID mismatch: this voice frame=" << ntohs(Frame->GetCodecPacket()->streamid) << " returned transcoder packet=" << ntohs(pack.streamid) << std::dec << std::noshowbase << std::endl; - if (pack.sequence != Frame->GetCodecPacket()->sequence) - std::cerr << "Sequence mismatch: this voice frame=" << Frame->GetCodecPacket()->sequence << " returned transcoder packet=" << pack.sequence << std::endl; + if (pack.streamid != Packet->GetCodecPacket()->streamid) + std::cerr << std::hex << std::showbase << "StreamID mismatch: this voice frame=" << ntohs(Packet->GetCodecPacket()->streamid) << " returned transcoder packet=" << ntohs(pack.streamid) << std::dec << std::noshowbase << std::endl; + if (pack.sequence != Packet->GetCodecPacket()->sequence) + std::cerr << "Sequence mismatch: this voice frame=" << Packet->GetCodecPacket()->sequence << " returned transcoder packet=" << pack.sequence << std::endl; } } else @@ -185,12 +197,10 @@ void CCodecStream::Task(void) } } - // anything in our queue - auto Packet = m_Queue.Pop(); - while (Packet) + // anything in our queue, then get it to the transcoder! + while (! m_Queue.IsEmpty()) { - // we need a CDvFramePacket pointer to access Frame stuff - auto Frame = (CDvFramePacket *)Packet.get(); + auto &Frame = m_Queue.Front(); if (m_IsOpen) { @@ -199,13 +209,22 @@ void CCodecStream::Task(void) Frame->SetTCParams(m_uiTotalPackets++); // now send to transcoder - g_Reflector.tcServer.Send(Frame->GetCodecPacket()); + int fd = g_TCServer.GetFD(Frame->GetCodecPacket()->module); + if (fd < 0) + { + // Crap! We've lost connection to the transcoder! + // we'll try to fix this on the next pass + return; + } - // push to our local queue where it can wait for the transcoder - m_LocalQueue.Push(std::move(Packet)); + if (g_TCServer.Send(fd, Frame->GetCodecPacket())) + { + // ditto, we'll try to fix this on the next pass + return; + } + // the fd was good and then the send was successful, so... + // push the frame to our local queue where it can wait for the transcoder + m_LocalQueue.Push(std::move(m_Queue.Pop())); } - - // get the next packet, if there is one - Packet = m_Queue.Pop(); } } diff --git a/reflector/CodecStream.h b/reflector/CodecStream.h index f23081b..51d2863 100644 --- a/reflector/CodecStream.h +++ b/reflector/CodecStream.h @@ -50,7 +50,7 @@ public: void Task(void); // pass-through - void Push(std::unique_ptr p) { m_Queue.Push(std::move(p)); } + void Push(std::unique_ptr p) { m_Queue.Push(std::move(p)); } protected: // identity @@ -67,7 +67,7 @@ protected: CPacketStream *m_PacketStream; // queues - CSafePacketQueue> m_LocalQueue, m_Queue; + CSafePacketQueue> m_LocalQueue, m_Queue; // thread std::atomic keep_running; diff --git a/reflector/Configure.cpp b/reflector/Configure.cpp index 9abe487..0fff6bc 100644 --- a/reflector/Configure.cpp +++ b/reflector/Configure.cpp @@ -305,6 +305,7 @@ bool CConfigure::ReadData(const std::string &path) } else badParam(key); + break; case ESection::modules: if (0 == key.compare(JMODULES)) { @@ -669,6 +670,12 @@ bool CConfigure::ReadData(const std::string &path) } } } + else + { + // there is no transcoder + data[g_Keys.tc.modules] = ""; + data[g_Keys.tc.bind] = ""; + } } // "simple" protocols with only a Port diff --git a/reflector/Global.h b/reflector/Global.h index 0a3c43c..0239507 100644 --- a/reflector/Global.h +++ b/reflector/Global.h @@ -21,6 +21,7 @@ #include "LookupDmr.h" #include "LookupNxdn.h" #include "LookupYsf.h" +#include "TCSocket.h" #include "JsonKeys.h" extern CReflector g_Reflector; @@ -31,3 +32,4 @@ extern CLookupDmr g_LDid; extern CLookupNxdn g_LNid; extern CLookupYsf g_LYtr; extern SJsonKeys g_Keys; +extern CTCServer g_TCServer; diff --git a/reflector/Main.cpp b/reflector/Main.cpp index 3d5e1a5..22cd1d4 100644 --- a/reflector/Main.cpp +++ b/reflector/Main.cpp @@ -32,6 +32,7 @@ CVersion g_Version(3,1,2); // The major byte should only change if the interl CLookupDmr g_LDid; CLookupNxdn g_LNid; CLookupYsf g_LYtr; +CTCServer g_TCServer; //////////////////////////////////////////////////////////////////////////////////////// diff --git a/reflector/PacketStream.cpp b/reflector/PacketStream.cpp index 7582c28..d728b56 100644 --- a/reflector/PacketStream.cpp +++ b/reflector/PacketStream.cpp @@ -92,9 +92,11 @@ void CPacketStream::Push(std::unique_ptr Packet) if ( m_CodecStream && Packet->IsDvFrame() && Packet->IsLocalOrigin()) { // yes, push packet to trancoder queue - // trancoder will push it after transcoding + // first, recast to a CDvFramePacket + auto Frame = std::unique_ptr(static_cast(Packet.release())); + // trancoder will push it to m_Queue after transcoding // is completed - m_CodecStream->Push(std::move(Packet)); + m_CodecStream->Push(std::move(Frame)); } else { diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index 08da97a..360a34c 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -70,7 +70,7 @@ bool CReflector::Start(void) // init transcoder comms if (port) - tcServer.Open(g_Configure.GetString(g_Keys.tc.bind), tcmods, port); + g_TCServer.Open(g_Configure.GetString(g_Keys.tc.bind), tcmods, port); // init gate keeper. It can only return true! g_GateKeeper.Init(); @@ -149,7 +149,7 @@ void CReflector::Stop(void) // stop transcoder comms // if it was never opened, then there is nothing to close; - tcServer.Close(); + g_TCServer.Close(); // stop & delete report threads if ( m_XmlReportFuture.valid() ) diff --git a/reflector/Reflector.h b/reflector/Reflector.h index 30f04d2..5d8fb52 100644 --- a/reflector/Reflector.h +++ b/reflector/Reflector.h @@ -25,7 +25,6 @@ #include "Peers.h" #include "Protocols.h" #include "PacketStream.h" -#include "TCTCPSocket.h" #ifndef NO_DHT #include "dht-values.h" @@ -84,9 +83,6 @@ public: void GetDHTConfig(const std::string &cs); #endif - // transcoder communication - CTCTCPServer tcServer; - protected: #ifndef NO_DHT // Publish DHT diff --git a/reflector/SafePacketQueue.h b/reflector/SafePacketQueue.h index 5b5ec60..6c5635e 100644 --- a/reflector/SafePacketQueue.h +++ b/reflector/SafePacketQueue.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -43,6 +44,18 @@ public: c.notify_one(); } + // You will die if the queue is empty! + T &Front(void) + { + std::lock_guard lock(m); + if (q.empty()) + { + std::cerr << "ERROR: CSavePacketQueue::Front() called, but queue is EMPTY!" << std::endl; + exit(1); + } + return q.front(); + } + T Pop(void) { std::lock_guard lock(m); diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp new file mode 100644 index 0000000..710e356 --- /dev/null +++ b/reflector/TCSocket.cpp @@ -0,0 +1,294 @@ +// urfd -- The universal reflector +// Copyright © 2024 Thomas A. Early N7TAE +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include +#include +#include +#include +#include +#include + +#include "IP.h" +#include "TCSocket.h" + +void CTCSocket::Close() +{ + std::lock_guard lck(m_Mutex); + for (auto item : m_FD) + close(item.second); + m_FD.clear(); +} + +void CTCSocket::Close(char mod) +{ + std::lock_guard lck(m_Mutex); + auto item = m_FD.find(mod); + if (m_FD.end() == item) + { + std::cerr << "Could not find a file descriptor for module '" << mod << "'" << std::endl; + return; + } + close(item->second); + m_FD.erase(item); +} + +void CTCSocket::Close(int fd) +{ + std::lock_guard lck(m_Mutex); + for (auto &p : m_FD) + { + if (fd == p.second) + { + close(fd); + m_FD.erase(p.first); + return; + } + } + std::cerr << "Could not find a file descriptor with a value of " << fd << std::endl; +} + +int CTCSocket::GetFD(char module) const +{ + std::lock_guard lck(m_Mutex); + const auto item = m_FD.find(module); + if (m_FD.cend() == item) + { + return -1; + } + return item->second; +} + +char CTCSocket::GetMod(int fd) const +{ + std::lock_guard lck(m_Mutex); + for (const auto &p : m_FD) + { + if (fd == p.second) + return p.first; + } + return '?'; +} + +bool CTCSocket::Send(int fd, const STCPacket *packet) +{ + unsigned count = 0; + auto data = (const unsigned char *)packet; + do { + auto n = send(fd, data+count, sizeof(STCPacket)-count, 0); + if (n <= 0) + { + if (0 == n) + { + std::cerr << "CTCSocket::Send: socket on module '" << packet->module << "' has been closed!" << std::endl; + } + else + { + perror("CTCSocket::Send"); + } + Close(packet->module); + return true; + } + count += n; + } while (count < sizeof(STCPacket)); + return false; +} + +bool CTCSocket::Receive(int fd, STCPacket *packet) +{ + auto data = (unsigned char *)packet; + auto n = recv(fd, data, sizeof(STCPacket), MSG_WAITALL); + if (n < 0) + { + perror("CTCSocket::Receive"); + Close(fd); + return true; + } + return n == sizeof(STCPacket); +} + +bool CTCServer::Open(const std::string &address, const std::string &modules, uint16_t port) +{ + m_Modules.assign(modules); + + CIp ip(address.c_str(), AF_UNSPEC, SOCK_STREAM, port); + + int fd = socket(ip.GetFamily(), SOCK_STREAM, 0); + if (fd < 0) + { + perror("Open socket"); + return true; + } + + int yes = 1; + int rv = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)); + if (rv < 0) + { + close(fd); + perror("Open setsockopt"); + return true; + } + + rv = bind(fd, ip.GetCPointer(), ip.GetSize()); + if (rv < 0) + { + close(fd); + perror("Open bind"); + return true; + } + + rv = listen(fd, 3); + if (rv < 0) + { + perror("Open listen"); + close(fd); + Close(); + return true; + } + + auto n = m_Modules.size(); + std::cout << "Waiting for " << n << " tC connection(s)..." << std::endl; + + while (m_FD.size() < n) + { + if (Accept()) + return true; + } + + m_listenSock = fd; + + return false; +} + +bool CTCServer::Accept() +{ + CIp their_addr; // connector's address information + + socklen_t sin_size = sizeof(struct sockaddr_storage); + + auto newfd = accept(m_listenSock, their_addr.GetPointer(), &sin_size); + if (newfd < 0) + { + if (EAGAIN == errno || EWOULDBLOCK == errno) + return false; + perror("accept"); + return true; + } + + char mod; + auto rv = recv(newfd, &mod, 1, 0); // retrieve the identification byte + if (rv != 1) + { + if (rv < 0) + perror("accept recv"); + else + std::cerr << "recv got no identification byte!" << std::endl; + close(newfd); + return true; + } + + if (std::string::npos == m_Modules.find(mod)) + { + std::cerr << "New connection for module '" << mod << "', but it's not configured!" << std::endl; + std::cerr << "The transcoded modules need to be configured identically for both urfd and tcd." << std::endl; + close(newfd); + return true; + } + + std::cout << "File descriptor " << newfd << " opened TCP port for module '" << mod << "' on " << their_addr << std::endl; + + std::lock_guard lck(m_Mutex); + m_FD[mod] = newfd; + + return false; +} + +bool CTCClient::Initialize(const std::string &address, const std::string &modules, uint16_t port) +{ + m_Address.assign(address); + m_Modules.assign(modules); + m_Port = port; + + std::cout << "Connecting to the TCP server..." << std::endl; + + for (char c : modules) + { + if (Connect(c)) + { + return true; + } + } + return false; +} + +bool CTCClient::Connect(char module) +{ + CIp ip(m_Address.c_str(), AF_UNSPEC, SOCK_STREAM, m_Port); + + auto fd = socket(ip.GetFamily(), SOCK_STREAM, 0); + if (fd < 0) + { + std::cerr << "Could not open socket for module '" << module << "'" << std::endl; + perror("TC client socket"); + return true; + } + + int yes = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int))) + { + perror("TC client setsockopt"); + close(fd); + return true; + } + + unsigned count = 0; + while (connect(fd, ip.GetCPointer(), ip.GetSize())) + { + if (ECONNREFUSED == errno) + { + if (0 == ++count % 100) std::cout << "Connection refused! Restart the server." << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + else + { + std::cerr << "For module '" << module << "' : "; + perror("connect"); + close(fd); + return true; + } + } + + int sent = send(fd, &module, 1, 0); // send the identification byte + if (sent < 0) + { + std::cerr << "Error sending ID byte to module '" << module << "':" << std::endl; + perror("send"); + close(fd); + return true; + } + else if (0 == sent) + { + std::cerr << "Could not set ID byte to module '" << module << "'" << std::endl; + close(fd); + return true; + } + + std::cout << "File descriptor " << fd << " on " << ip << " opened for module '" << module << "'" << std::endl; + + std::lock_guard lck(m_Mutex); + m_FD[module] = fd; + + return false; +} diff --git a/reflector/TCTCPSocket.h b/reflector/TCSocket.h similarity index 62% rename from reflector/TCTCPSocket.h rename to reflector/TCSocket.h index 2cce5a5..574528d 100644 --- a/reflector/TCTCPSocket.h +++ b/reflector/TCSocket.h @@ -18,43 +18,53 @@ #include #include +#include #include #include "TCPacketDef.h" -class CTCTCPSocket +class CTCSocket { public: - CTCTCPSocket() {} - virtual ~CTCTCPSocket() { Close(); } + CTCSocket() {} + virtual ~CTCSocket() { Close(); } void Close(); // close all open sockets + void Close(char module); // close a specific module + void Close(int fd); // close a specific file descriptor // bool functions return true on failure - virtual bool Open(const std::string &address, const std::string &modules, uint16_t port) = 0; - bool Send(const STCPacket *packet); + bool Send(int fd, const STCPacket *packet); bool Receive(int fd, STCPacket *packet); int GetFD(char module) const; // can return -1! + char GetMod(int fd) const; protected: - void Close(char); // close a specific module - std::unordered_map m_FD; + mutable std::mutex m_Mutex; }; -class CTCTCPServer : public CTCTCPSocket +class CTCServer : public CTCSocket { public: - CTCTCPServer() : CTCTCPSocket() {} - ~CTCTCPServer() {} + CTCServer() : CTCSocket() {} + ~CTCServer() {} bool Open(const std::string &address, const std::string &modules, uint16_t port); + bool Accept(); +private: + int m_listenSock; + std::string m_Modules; }; -class CTCTCPClient : public CTCTCPSocket +class CTCClient : public CTCSocket { public: - CTCTCPClient() : CTCTCPSocket() {} - ~CTCTCPClient() {} - bool Open(const std::string &address, const std::string &modules, uint16_t port); + CTCClient() : CTCSocket(), m_Port(0) {} + ~CTCClient() {} + bool Initialize(const std::string &address, const std::string &modules, uint16_t port); + bool Connect(char module); +private: + std::string m_Address, m_Modules; + uint16_t m_Port; }; diff --git a/reflector/TCTCPSocket.cpp b/reflector/TCTCPSocket.cpp deleted file mode 100644 index 3bbcabb..0000000 --- a/reflector/TCTCPSocket.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// urfd -- The universal reflector -// Copyright © 2024 Thomas A. Early N7TAE -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#include -#include -#include -#include -#include -#include - -#include "IP.h" -#include "TCTCPSocket.h" - -void CTCTCPSocket::Close() -{ - for (auto item : m_FD) - close(item.second); - m_FD.clear(); -} - -void CTCTCPSocket::Close(char mod) -{ - auto item = m_FD.find(mod); - if (m_FD.end() == item) - return; - - close(item->second); - m_FD.erase(item); -} - -bool CTCTCPSocket::Send(const STCPacket *packet) -{ - int fd = GetFD(packet->module); - if (fd < 0) - { - return true; - } - - long count = 0; - auto data = (const unsigned char *)packet; - do { - auto n = send(fd, data+count, sizeof(STCPacket)-count, 0); - if (n <= 0) - { - if (0 == n) - { - std::cerr << "CTCTCPSocket::Send: socket on module '" << packet->module << "' has been closed!" << std::endl; - } - else - { - perror("CTCTCPSocket::Send"); - } - Close(packet->module); - return true; - } - count += n; - } while (count < sizeof(STCPacket)); - return false; -} - -bool CTCTCPSocket::Receive(int fd, STCPacket *packet) -{ - auto data = (unsigned char *)packet; - auto n = recv(fd, data, sizeof(STCPacket), MSG_WAITALL); - if (n < 0) - { - perror("CTCTCPSocket::Receive"); - return true; - } - return n == sizeof(STCPacket); -} - -int CTCTCPSocket::GetFD(char module) const -{ - const auto item = m_FD.find(module); - if (m_FD.cend() == item) - { - return -1; - } - return item->second; -} - -bool CTCTCPServer::Open(const std::string &address, const std::string &tcmodules, uint16_t port) -{ - int fd; - CIp ip(address.c_str(), AF_UNSPEC, SOCK_STREAM, port); - - fd = socket(ip.GetFamily(), SOCK_STREAM, 0); - if (fd < 0) - { - perror("Open socket"); - return true; - } - - int yes = 1; - int rv = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)); - if (rv < 0) - { - close(fd); - perror("Open setsockopt"); - return true; - } - - rv = bind(fd, ip.GetCPointer(), ip.GetSize()); - if (rv < 0) - { - close(fd); - perror("Open bind"); - return true; - } - - rv = listen(fd, 3); - if (rv < 0) - { - perror("Open listen"); - close(fd); - Close(); - return true; - } - - std::cout << "Waiting for " << tcmodules.size() << " transcoder connection(s)..." << std::endl; - for (unsigned x=0; xfirst << " has no module '" << c << "'" << std::endl; } - else if ((std::string::npos == it->second.GetTCMods().find(c)) != (std::string::npos == g_Configure.GetString(g_Keys.modules.tcmodules).find(c))) + else if ((std::string::npos == it->second.GetTCMods().find(c)) != (std::string::npos == g_Configure.GetString(g_Keys.tc.modules).find(c))) { // are the transcoding states on both sides mismatched? ok = false; std::cerr << "The transcode states for module '" << c << "' don't match for this reflector and " << it->first << std::endl; From 38984be6bb1e4a53928cbf9bc4553e197c79757a Mon Sep 17 00:00:00 2001 From: Tom Early Date: Wed, 8 May 2024 16:41:07 -0700 Subject: [PATCH 15/41] add select to Accept --- reflector/CodecStream.cpp | 10 ++++++---- reflector/TCSocket.cpp | 32 ++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index bde8508..2a436e2 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -110,17 +110,19 @@ void CCodecStream::Thread() void CCodecStream::Task(void) { int fd = g_TCServer.GetFD(m_CSModule); + + // if the fd is not good we need to reestablish it if (fd < 0) // log the situation - std::cout << "Lost connection to transcoder, module '" << m_CSModule << "', waiting..." << std::endl; + std::cout << "Lost connection to transcoder, module '" << m_CSModule << "', waiting for new connection..." << std::endl; while (fd < 0) { - if (g_TCServer.Accept()) // we will block until we have a new connection + if (g_TCServer.Accept()) // try to get a connection { std::cerr << "UNRECOVERABLE ERROR!" << std::endl; exit(1); } - // Accept was sucessful, but we need to make sure THIS MODULE was reestablished - // It's possile that other Transcoder ports were lost + // Either Accept timed out, or it's possile that other Transcoder ports were instead reopened + // So we'll check to see if the one for this module is okay now fd = g_TCServer.GetFD(m_CSModule); } diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 710e356..03cab1a 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include "IP.h" #include "TCSocket.h" @@ -159,7 +160,7 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin } auto n = m_Modules.size(); - std::cout << "Waiting for " << n << " tC connection(s)..." << std::endl; + std::cout << "Waiting for " << n << " transcoder connection(s)..." << std::endl; while (m_FD.size() < n) { @@ -174,6 +175,29 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin bool CTCServer::Accept() { + struct timeval tv; + fd_set readfds; + + // 10 milliseconds + tv.tv_sec = 0; + tv.tv_usec = 10000; + + FD_ZERO(&readfds); + FD_SET(m_listenSock, &readfds); + + // don't care about writefds and exceptfds: + int rv = select(m_listenSock+1, &readfds, NULL, NULL, &tv); + if (rv < 0) + { + perror("Accept select"); + return true; + } + + if (0 == rv) // we timed out waiting for something + return false; + + // here comes a connect + CIp their_addr; // connector's address information socklen_t sin_size = sizeof(struct sockaddr_storage); @@ -183,16 +207,16 @@ bool CTCServer::Accept() { if (EAGAIN == errno || EWOULDBLOCK == errno) return false; - perror("accept"); + perror("Accept accept"); return true; } char mod; - auto rv = recv(newfd, &mod, 1, 0); // retrieve the identification byte + rv = recv(newfd, &mod, 1, 0); // retrieve the identification byte if (rv != 1) { if (rv < 0) - perror("accept recv"); + perror("Accept recv"); else std::cerr << "recv got no identification byte!" << std::endl; close(newfd); From e0f3797d69e4dd5e44f8cf42b6882586de28fbe1 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Thu, 9 May 2024 13:02:27 -0700 Subject: [PATCH 16/41] use poll in TCSocket --- reflector/CodecStream.cpp | 18 +---- reflector/TCSocket.cpp | 148 +++++++++++++++++++++++--------------- reflector/TCSocket.h | 19 +++-- 3 files changed, 103 insertions(+), 82 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index 2a436e2..4760b0f 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -18,7 +18,6 @@ #include -#include #include "Global.h" #include "DVFramePacket.h" @@ -114,6 +113,7 @@ void CCodecStream::Task(void) // if the fd is not good we need to reestablish it if (fd < 0) // log the situation std::cout << "Lost connection to transcoder, module '" << m_CSModule << "', waiting for new connection..." << std::endl; + while (fd < 0) { if (g_TCServer.Accept()) // try to get a connection @@ -127,20 +127,8 @@ void CCodecStream::Task(void) } STCPacket pack; - struct timeval tv; - fd_set readfds; - - tv.tv_sec = 0; - tv.tv_usec = 7000; - - FD_ZERO(&readfds); - FD_SET(fd, &readfds); - - // don't care about writefds and exceptfds: - if (select(fd+1, &readfds, NULL, NULL, &tv)) + if (g_TCServer.Receive(m_CSModule, &pack, 8)) { - if (g_TCServer.Receive(fd, &pack)) - return; // update statistics double rt = pack.rt_timer.time(); // the round-trip time if (0 == m_RTCount) @@ -219,7 +207,7 @@ void CCodecStream::Task(void) return; } - if (g_TCServer.Send(fd, Frame->GetCodecPacket())) + if (g_TCServer.Send(Frame->GetCodecPacket())) { // ditto, we'll try to fix this on the next pass return; diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 03cab1a..61fd109 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -27,34 +27,36 @@ void CTCSocket::Close() { - std::lock_guard lck(m_Mutex); - for (auto item : m_FD) - close(item.second); - m_FD.clear(); + for (auto &item : m_Pfd) + { + if (item.fd >= 0) + { + close(item.fd); + } + } + m_Pfd.clear(); } void CTCSocket::Close(char mod) { - std::lock_guard lck(m_Mutex); - auto item = m_FD.find(mod); - if (m_FD.end() == item) + auto pos = m_Modules.find(mod); + if (std::string::npos == pos) { - std::cerr << "Could not find a file descriptor for module '" << mod << "'" << std::endl; + std::cerr << "Could not find module '" << mod << "'" << std::endl; return; } - close(item->second); - m_FD.erase(item); + close(m_Pfd[pos].fd); + m_Pfd[pos].fd = -1; } void CTCSocket::Close(int fd) { - std::lock_guard lck(m_Mutex); - for (auto &p : m_FD) + for (auto &p : m_Pfd) { - if (fd == p.second) + if (fd == p.fd) { - close(fd); - m_FD.erase(p.first); + close(p.fd); + p.fd = -1; return; } } @@ -63,32 +65,47 @@ void CTCSocket::Close(int fd) int CTCSocket::GetFD(char module) const { - std::lock_guard lck(m_Mutex); - const auto item = m_FD.find(module); - if (m_FD.cend() == item) - { + auto pos = m_Modules.find(module); + if (std::string::npos == pos) return -1; - } - return item->second; + return m_Pfd[pos].fd; } char CTCSocket::GetMod(int fd) const { - std::lock_guard lck(m_Mutex); - for (const auto &p : m_FD) + for (unsigned i=0; i m_Pfd[i].fd) + return true; + } + return false; +} + +bool CTCSocket::Send(const STCPacket *packet) { + const auto pos = m_Modules.find(packet->module); + if (pos == std::string::npos) + { + std::cerr << "Can't Send() this packet to unconfigured module '" << packet->module << "'" << std::endl; + return true; + } unsigned count = 0; auto data = (const unsigned char *)packet; do { - auto n = send(fd, data+count, sizeof(STCPacket)-count, 0); + auto n = send(m_Pfd[pos].fd, data+count, sizeof(STCPacket)-count, 0); if (n <= 0) { if (0 == n) @@ -107,22 +124,47 @@ bool CTCSocket::Send(int fd, const STCPacket *packet) return false; } -bool CTCSocket::Receive(int fd, STCPacket *packet) +// returns true if successful +bool CTCServer::Receive(char module, STCPacket *packet, int ms) { + const auto pos = m_Modules.find(module); + if (pos == std::string::npos) + { + std::cerr << "Can't Recevieve on unconfigured module '" << module << "'" << std::endl; + return false; + } + + auto rv = poll(&m_Pfd[pos], 1, ms); + if (rv < 0) + { + perror("Recieve poll"); + Close(m_Pfd[pos].fd); + return false; + } + auto data = (unsigned char *)packet; - auto n = recv(fd, data, sizeof(STCPacket), MSG_WAITALL); + auto n = recv(m_Pfd[pos].fd, data, sizeof(STCPacket), MSG_WAITALL); if (n < 0) { - perror("CTCSocket::Receive"); - Close(fd); - return true; + perror("Receive recv"); + Close(m_Pfd[pos].fd); + return false; } - return n == sizeof(STCPacket); + if (n != sizeof(STCPacket)) + std::cout << "Warning: Receive only read " << n << " bytes of the transcoder packet!" << std::endl; + + return true; } bool CTCServer::Open(const std::string &address, const std::string &modules, uint16_t port) { m_Modules.assign(modules); + m_Pfd.resize(m_Modules.size()+1); + for (auto &pf : m_Pfd) + { + pf.fd = -1; + pf.events = POLLIN; + } CIp ip(address.c_str(), AF_UNSPEC, SOCK_STREAM, port); @@ -162,51 +204,38 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin auto n = m_Modules.size(); std::cout << "Waiting for " << n << " transcoder connection(s)..." << std::endl; - while (m_FD.size() < n) + while (any_are_closed()) { if (Accept()) return true; } - m_listenSock = fd; + m_Pfd.back().fd = fd; return false; } bool CTCServer::Accept() { - struct timeval tv; - fd_set readfds; - - // 10 milliseconds - tv.tv_sec = 0; - tv.tv_usec = 10000; - - FD_ZERO(&readfds); - FD_SET(m_listenSock, &readfds); - - // don't care about writefds and exceptfds: - int rv = select(m_listenSock+1, &readfds, NULL, NULL, &tv); + auto rv = poll(&m_Pfd.back(), 1, 10); if (rv < 0) { - perror("Accept select"); + perror("Accept poll"); return true; } if (0 == rv) // we timed out waiting for something return false; - // here comes a connect + // rv has to be 1, so, here comes a connect request CIp their_addr; // connector's address information socklen_t sin_size = sizeof(struct sockaddr_storage); - auto newfd = accept(m_listenSock, their_addr.GetPointer(), &sin_size); + auto newfd = accept(m_Pfd.back().fd, their_addr.GetPointer(), &sin_size); if (newfd < 0) { - if (EAGAIN == errno || EWOULDBLOCK == errno) - return false; perror("Accept accept"); return true; } @@ -223,7 +252,8 @@ bool CTCServer::Accept() return true; } - if (std::string::npos == m_Modules.find(mod)) + const auto pos = m_Modules.find(mod); + if (std::string::npos == pos) { std::cerr << "New connection for module '" << mod << "', but it's not configured!" << std::endl; std::cerr << "The transcoded modules need to be configured identically for both urfd and tcd." << std::endl; @@ -233,8 +263,7 @@ bool CTCServer::Accept() std::cout << "File descriptor " << newfd << " opened TCP port for module '" << mod << "' on " << their_addr << std::endl; - std::lock_guard lck(m_Mutex); - m_FD[mod] = newfd; + m_Pfd[pos].fd = newfd; return false; } @@ -259,6 +288,12 @@ bool CTCClient::Initialize(const std::string &address, const std::string &module bool CTCClient::Connect(char module) { + const auto pos = m_Modules.find(module); + if (pos == std::string::npos) + { + std::cerr << "CTCClient::Connect: could not find module '" << module << "' in configured modules!" << std::endl; + return true; + } CIp ip(m_Address.c_str(), AF_UNSPEC, SOCK_STREAM, m_Port); auto fd = socket(ip.GetFamily(), SOCK_STREAM, 0); @@ -311,8 +346,7 @@ bool CTCClient::Connect(char module) std::cout << "File descriptor " << fd << " on " << ip << " opened for module '" << module << "'" << std::endl; - std::lock_guard lck(m_Mutex); - m_FD[module] = fd; + m_Pfd[pos].fd = fd; return false; } diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 574528d..8b6b484 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -19,7 +19,8 @@ #include #include #include -#include +#include +#include #include "TCPacketDef.h" @@ -33,16 +34,16 @@ public: void Close(char module); // close a specific module void Close(int fd); // close a specific file descriptor - // bool functions return true on failure - bool Send(int fd, const STCPacket *packet); - bool Receive(int fd, STCPacket *packet); + // most bool functions return true on failure + bool Send(const STCPacket *packet); int GetFD(char module) const; // can return -1! char GetMod(int fd) const; protected: - std::unordered_map m_FD; - mutable std::mutex m_Mutex; + bool any_are_closed(); + std::vector m_Pfd; + std::string m_Modules; }; class CTCServer : public CTCSocket @@ -52,9 +53,7 @@ public: ~CTCServer() {} bool Open(const std::string &address, const std::string &modules, uint16_t port); bool Accept(); -private: - int m_listenSock; - std::string m_Modules; + bool Receive(char module, STCPacket *packet, int ms); }; class CTCClient : public CTCSocket @@ -65,6 +64,6 @@ public: bool Initialize(const std::string &address, const std::string &modules, uint16_t port); bool Connect(char module); private: - std::string m_Address, m_Modules; + std::string m_Address; uint16_t m_Port; }; From 37d47c1ac6fb5db0a7ec4bbb3e5236d8a3478c9c Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 10 May 2024 14:48:13 -0700 Subject: [PATCH 17/41] Changes for CTCPClient --- reflector/CodecStream.cpp | 2 +- reflector/TCSocket.cpp | 113 ++++++++++++++++++++++++++++++++------ reflector/TCSocket.h | 4 ++ 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index 4760b0f..0df5425 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -122,7 +122,7 @@ void CCodecStream::Task(void) exit(1); } // Either Accept timed out, or it's possile that other Transcoder ports were instead reopened - // So we'll check to see if the one for this module is okay now + // So we'll check to see if this module is now open fd = g_TCServer.GetFD(m_CSModule); } diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 61fd109..b4c3995 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -31,7 +31,7 @@ void CTCSocket::Close() { if (item.fd >= 0) { - close(item.fd); + Close(item.fd); } } m_Pfd.clear(); @@ -45,7 +45,7 @@ void CTCSocket::Close(char mod) std::cerr << "Could not find module '" << mod << "'" << std::endl; return; } - close(m_Pfd[pos].fd); + Close(m_Pfd[pos].fd); m_Pfd[pos].fd = -1; } @@ -55,8 +55,13 @@ void CTCSocket::Close(int fd) { if (fd == p.fd) { - close(p.fd); - p.fd = -1; + if (close(p.fd)) + { + std::cerr << "Error while closing " << fd << ": "; + perror("close"); + } + else + p.fd = -1; return; } } @@ -124,7 +129,7 @@ bool CTCSocket::Send(const STCPacket *packet) return false; } -// returns true if successful +// returns true if data is returned bool CTCServer::Receive(char module, STCPacket *packet, int ms) { const auto pos = m_Modules.find(module); @@ -134,24 +139,38 @@ bool CTCServer::Receive(char module, STCPacket *packet, int ms) return false; } - auto rv = poll(&m_Pfd[pos], 1, ms); + auto fds = &m_Pfd[pos]; + auto rv = poll(fds, 1, ms); if (rv < 0) { - perror("Recieve poll"); - Close(m_Pfd[pos].fd); + perror("CTCServer::Recieve poll"); + Close(fds->fd); return false; } - auto data = (unsigned char *)packet; - auto n = recv(m_Pfd[pos].fd, data, sizeof(STCPacket), MSG_WAITALL); - if (n < 0) + if (0 == rv) + return false; + + if (fds->revents & POLLIN) { - perror("Receive recv"); - Close(m_Pfd[pos].fd); + auto data = (unsigned char *)packet; + auto n = recv(fds->fd, data, sizeof(STCPacket), MSG_WAITALL); + if (n < 0) + { + perror("Receive recv"); + Close(fds->fd); + return false; + } + if (n != sizeof(STCPacket)) + std::cout << "WARNING: Receive only read " << n << " bytes of the transcoder packet!" << std::endl; + } + + if (fds->revents & POLLERR || fds->revents & POLLHUP) + { + std::cerr << ((fds->revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on fd " << fds->fd << ", closing socket" << std::endl; + Close(fds->fd); return false; } - if (n != sizeof(STCPacket)) - std::cout << "Warning: Receive only read " << n << " bytes of the transcoder packet!" << std::endl; return true; } @@ -317,7 +336,7 @@ bool CTCClient::Connect(char module) { if (ECONNREFUSED == errno) { - if (0 == ++count % 100) std::cout << "Connection refused! Restart the server." << std::endl; + if (0 == ++count % 100) std::cout << "Connection refused! Restart the system." << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } else @@ -350,3 +369,65 @@ bool CTCClient::Connect(char module) return false; } + +bool CTCClient::CheckConnections() +{ + for (char m : m_Modules) + { + if (-1 == GetFD(m)) + { + if (Connect(m)) + { + return true; + } + } + } + return false; +} + +// returns true if data is returned. +// queue should be initially empty! +bool CTCClient::Receive(std::queue> &queue, int ms) +{ + auto n = m_Pfd.size(); + auto rv = poll(m_Pfd.data(), n, ms); + if (0 == rv) + return false; + + if (rv < 0) + { + perror("CTCClient::Receive poll"); + return false; + } + + for (auto &pfd : m_Pfd) + { + if (pfd.revents & POLLIN) + { + auto packet = std::make_unique(); + auto data = (unsigned char *)packet.get(); + auto n = recv(pfd.fd, data, sizeof(STCPacket), MSG_WAITALL); + + if (n < 0) + { + perror("Receive recv"); + Close(pfd.fd); + packet.reset(); + } + else if (n > 0) + { + if (n != sizeof(STCPacket)) + std::cout << "WARNING: Receive only read " << n << " bytes of the transcoder packet!" << std::endl; + + queue.push(std::move(packet)); + } + } + + if (pfd.revents & POLLERR || pfd.revents & POLLHUP) + { + std::cerr << ((pfd.revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on fd " << pfd.fd << ", closing socket" << std::endl; + Close(pfd.fd); + } + } + return ! queue.empty(); +} diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 8b6b484..57d784c 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include "TCPacketDef.h" @@ -63,6 +65,8 @@ public: ~CTCClient() {} bool Initialize(const std::string &address, const std::string &modules, uint16_t port); bool Connect(char module); + bool CheckConnections(); + bool Receive(std::queue> &queue, int ms); private: std::string m_Address; uint16_t m_Port; From cf86b3984843422faca158c3abfe92b5215ea150 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 13 May 2024 11:29:40 -0700 Subject: [PATCH 18/41] Configure bug fix --- reflector/Configure.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reflector/Configure.cpp b/reflector/Configure.cpp index 0fff6bc..fa29100 100644 --- a/reflector/Configure.cpp +++ b/reflector/Configure.cpp @@ -290,10 +290,10 @@ bool CConfigure::ReadData(const std::string &path) break; case ESection::tc: if (0 == key.compare(JPORT)) - data[g_Keys.tc.port] = getUnsigned(value, "Transcoder Port", 0, 40000, 10100); - else if (key.compare(JBINDINGADDRESS)) + data[g_Keys.tc.port] = getUnsigned(value, "Transcoder Port", 1024, 40000, 10100); + else if (0 == key.compare(JBINDINGADDRESS)) data[g_Keys.tc.bind] = value; - else if (key.compare(JMODULES)) + else if (0 == key.compare(JMODULES)) { std::string m(value); if (checkModules(m)) From bcd9f5cdae08234c2acf50ed019d16b708be90b8 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 13 May 2024 15:59:05 -0700 Subject: [PATCH 19/41] forgot to init the vector --- reflector/TCSocket.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index b4c3995..b29e02c 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -293,6 +293,13 @@ bool CTCClient::Initialize(const std::string &address, const std::string &module m_Modules.assign(modules); m_Port = port; + m_Pfd.resize(m_Modules.size()); + for (auto &pf : m_Pfd) + { + pf.fd = -1; + pf.events = POLLIN; + } + std::cout << "Connecting to the TCP server..." << std::endl; for (char c : modules) From 4d8582c5c2bea2af931c18b88cdfd2a72b4ba3d3 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 13 May 2024 18:05:33 -0700 Subject: [PATCH 20/41] server open bug --- reflector/TCSocket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index b29e02c..d968a90 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -222,6 +222,7 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin auto n = m_Modules.size(); std::cout << "Waiting for " << n << " transcoder connection(s)..." << std::endl; + m_Pfd.back().fd = fd; while (any_are_closed()) { @@ -229,7 +230,6 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin return true; } - m_Pfd.back().fd = fd; return false; } From 8698c70ab806c5234b19cc7d659cc15e7736fba8 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Tue, 14 May 2024 18:28:26 -0700 Subject: [PATCH 21/41] some redesign of TC socket --- reflector/CodecStream.cpp | 2 +- reflector/TCSocket.cpp | 100 +++++++++++++++++++------------------- reflector/TCSocket.h | 15 ++++-- 3 files changed, 61 insertions(+), 56 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index 0df5425..c3693d1 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -127,7 +127,7 @@ void CCodecStream::Task(void) } STCPacket pack; - if (g_TCServer.Receive(m_CSModule, &pack, 8)) + if (! g_TCServer.Receive(m_CSModule, pack, 8)) { // update statistics double rt = pack.rt_timer.time(); // the round-trip time diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index d968a90..ef62f91 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -88,7 +88,7 @@ char CTCSocket::GetMod(int fd) const return '?'; } -bool CTCSocket::any_are_closed() +bool CTCServer::any_are_closed() { const auto n = m_Modules.size(); for (unsigned i=0; ifd); - return false; + return true; } if (0 == rv) - return false; + return false; // timeout if (fds->revents & POLLIN) { - auto data = (unsigned char *)packet; - auto n = recv(fds->fd, data, sizeof(STCPacket), MSG_WAITALL); - if (n < 0) - { - perror("Receive recv"); - Close(fds->fd); - return false; - } - if (n != sizeof(STCPacket)) - std::cout << "WARNING: Receive only read " << n << " bytes of the transcoder packet!" << std::endl; + return receive(fds->fd, packet); } if (fds->revents & POLLERR || fds->revents & POLLHUP) { std::cerr << ((fds->revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on fd " << fds->fd << ", closing socket" << std::endl; Close(fds->fd); - return false; + return true; } - return true; + return false; } bool CTCServer::Open(const std::string &address, const std::string &modules, uint16_t port) @@ -220,21 +225,24 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin return true; } - auto n = m_Modules.size(); - std::cout << "Waiting for " << n << " transcoder connection(s)..." << std::endl; m_Pfd.back().fd = fd; + std::cout << "Waiting for " << m_Modules.size() << " transcoder connection(s) on fd " << fd << "..." << std::endl; + return Accept(); +} + +bool CTCServer::Accept() +{ while (any_are_closed()) { - if (Accept()) + if (AcceptOne()) return true; } - return false; } -bool CTCServer::Accept() +bool CTCServer::AcceptOne() { auto rv = poll(&m_Pfd.back(), 1, 10); if (rv < 0) @@ -243,8 +251,8 @@ bool CTCServer::Accept() return true; } - if (0 == rv) // we timed out waiting for something - return false; + if (0 == rv) + return false; // timeout // rv has to be 1, so, here comes a connect request @@ -260,7 +268,7 @@ bool CTCServer::Accept() } char mod; - rv = recv(newfd, &mod, 1, 0); // retrieve the identification byte + rv = recv(newfd, &mod, 1, 0); // block to get the identification byte if (rv != 1) { if (rv < 0) @@ -377,62 +385,54 @@ bool CTCClient::Connect(char module) return false; } -bool CTCClient::CheckConnections() +bool CTCClient::ReConnect() { + bool rv = false; for (char m : m_Modules) { if (-1 == GetFD(m)) { if (Connect(m)) { - return true; + rv = true; } } } - return false; + return rv; } -// returns true if data is returned. -// queue should be initially empty! bool CTCClient::Receive(std::queue> &queue, int ms) { - auto n = m_Pfd.size(); + const auto n = m_Pfd.size(); auto rv = poll(m_Pfd.data(), n, ms); - if (0 == rv) - return false; if (rv < 0) { - perror("CTCClient::Receive poll"); + perror("Receive poll"); return false; } + if (0 == rv) + return false; + for (auto &pfd : m_Pfd) { if (pfd.revents & POLLIN) { - auto packet = std::make_unique(); - auto data = (unsigned char *)packet.get(); - auto n = recv(pfd.fd, data, sizeof(STCPacket), MSG_WAITALL); - - if (n < 0) + auto p_tcpack = std::make_unique(); + if (receive(pfd.fd, *p_tcpack)) { - perror("Receive recv"); - Close(pfd.fd); - packet.reset(); + p_tcpack.reset(); } - else if (n > 0) + else { - if (n != sizeof(STCPacket)) - std::cout << "WARNING: Receive only read " << n << " bytes of the transcoder packet!" << std::endl; - - queue.push(std::move(packet)); + queue.push(std::move(p_tcpack)); } } if (pfd.revents & POLLERR || pfd.revents & POLLHUP) { - std::cerr << ((pfd.revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on fd " << pfd.fd << ", closing socket" << std::endl; + std::cerr << "IO ERROR on Receive module " << GetMod(pfd.fd) << std::endl; Close(pfd.fd); } } diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 57d784c..47b98bd 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -35,15 +35,15 @@ public: void Close(); // close all open sockets void Close(char module); // close a specific module void Close(int fd); // close a specific file descriptor + bool receive(int fd, STCPacket &packet); - // most bool functions return true on failure + // All bool functions return true if there was an error bool Send(const STCPacket *packet); int GetFD(char module) const; // can return -1! char GetMod(int fd) const; protected: - bool any_are_closed(); std::vector m_Pfd; std::string m_Modules; }; @@ -54,8 +54,12 @@ public: CTCServer() : CTCSocket() {} ~CTCServer() {} bool Open(const std::string &address, const std::string &modules, uint16_t port); + bool Receive(char module, STCPacket &packet, int ms); bool Accept(); - bool Receive(char module, STCPacket *packet, int ms); + +private: + bool any_are_closed(); + bool AcceptOne(); }; class CTCClient : public CTCSocket @@ -64,10 +68,11 @@ public: CTCClient() : CTCSocket(), m_Port(0) {} ~CTCClient() {} bool Initialize(const std::string &address, const std::string &modules, uint16_t port); - bool Connect(char module); - bool CheckConnections(); bool Receive(std::queue> &queue, int ms); + bool ReConnect(); + private: + bool Connect(char module); std::string m_Address; uint16_t m_Port; }; From e715f57effceb7fe44679221bf9603be18462008 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Thu, 16 May 2024 14:27:42 -0700 Subject: [PATCH 22/41] tweaking how the TC socket deals with errors --- reflector/CodecStream.cpp | 2 +- reflector/TCSocket.cpp | 14 ++++++++------ reflector/TCSocket.h | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index c3693d1..a84f6aa 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -118,7 +118,7 @@ void CCodecStream::Task(void) { if (g_TCServer.Accept()) // try to get a connection { - std::cerr << "UNRECOVERABLE ERROR!" << std::endl; + std::cerr << "Unrecoverable ERROR! Quiting..." << std::endl; exit(1); } // Either Accept timed out, or it's possile that other Transcoder ports were instead reopened diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index ef62f91..3297fdd 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -227,7 +227,7 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin m_Pfd.back().fd = fd; - std::cout << "Waiting for " << m_Modules.size() << " transcoder connection(s) on fd " << fd << "..." << std::endl; + std::cout << "Waiting for " << m_Modules.size() << " transcoder connection(s) on " << ip << "..." << std::endl; return Accept(); } @@ -235,14 +235,14 @@ bool CTCServer::Accept() { while (any_are_closed()) { - if (AcceptOne()) + if (acceptone()) return true; } return false; } -bool CTCServer::AcceptOne() +bool CTCServer::acceptone() { auto rv = poll(&m_Pfd.back(), 1, 10); if (rv < 0) @@ -295,7 +295,7 @@ bool CTCServer::AcceptOne() return false; } -bool CTCClient::Initialize(const std::string &address, const std::string &modules, uint16_t port) +bool CTCClient::Open(const std::string &address, const std::string &modules, uint16_t port) { m_Address.assign(address); m_Modules.assign(modules); @@ -409,12 +409,13 @@ bool CTCClient::Receive(std::queue> &queue, int ms) if (rv < 0) { perror("Receive poll"); - return false; + return true; } if (0 == rv) return false; + bool some_closed = false; for (auto &pfd : m_Pfd) { if (pfd.revents & POLLIN) @@ -434,7 +435,8 @@ bool CTCClient::Receive(std::queue> &queue, int ms) { std::cerr << "IO ERROR on Receive module " << GetMod(pfd.fd) << std::endl; Close(pfd.fd); + some_closed = true; } } - return ! queue.empty(); + return some_closed; } diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 47b98bd..cf1f269 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -32,6 +32,7 @@ public: CTCSocket() {} virtual ~CTCSocket() { Close(); } + virtual bool Open(const std::string &address, const std::string &modules, uint16_t port) = 0; void Close(); // close all open sockets void Close(char module); // close a specific module void Close(int fd); // close a specific file descriptor @@ -59,7 +60,7 @@ public: private: bool any_are_closed(); - bool AcceptOne(); + bool acceptone(); }; class CTCClient : public CTCSocket @@ -67,7 +68,7 @@ class CTCClient : public CTCSocket public: CTCClient() : CTCSocket(), m_Port(0) {} ~CTCClient() {} - bool Initialize(const std::string &address, const std::string &modules, uint16_t port); + bool Open(const std::string &address, const std::string &modules, uint16_t port); bool Receive(std::queue> &queue, int ms); bool ReConnect(); From 45cf750bdd1f8682f5bd10841943c5ca9d0ebf9b Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 17 May 2024 13:30:16 -0700 Subject: [PATCH 23/41] more tweaks for TC socket --- reflector/TCSocket.cpp | 92 +++++++++++++++++++++++------------------- reflector/TCSocket.h | 4 +- 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 3297fdd..3699dae 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -22,7 +22,6 @@ #include #include -#include "IP.h" #include "TCSocket.h" void CTCSocket::Close() @@ -90,10 +89,9 @@ char CTCSocket::GetMod(int fd) const bool CTCServer::any_are_closed() { - const auto n = m_Modules.size(); - for (unsigned i=0; i m_Pfd[i].fd) + if (0 > fds.fd) return true; } return false; @@ -138,8 +136,16 @@ bool CTCSocket::receive(int fd, STCPacket &packet) Close(fd); return true; } + + if (0 == n) + { + std::cerr << "recv() returned zero bytes from mdule '" << GetMod(fd) << "'"; + Close(fd); + return true; + } + if (n != sizeof(STCPacket)) - std::cout << "WARNING: Receive only read " << n << " bytes of the transcoder packet!" << std::endl; + std::cout << "WARNING: Receive only read " << n << " bytes of the transcoder packet from module '" << GetMod(fd) << "'" << std::endl; return false; } @@ -149,31 +155,37 @@ bool CTCServer::Receive(char module, STCPacket &packet, int ms) const auto pos = m_Modules.find(module); if (pos == std::string::npos) { - std::cerr << "Can't Recevieve on unconfigured module '" << module << "'" << std::endl; + std::cerr << "Can't receive on unconfigured module '" << module << "'" << std::endl; + return true; + } + + auto pfds = &m_Pfd[pos]; + if (pfds->fd < 0) + { + std::cerr << "Can't receive on module '" << module << "' because it's closed" << std::endl; return true; } - auto fds = &m_Pfd[pos]; - auto rv = poll(fds, 1, ms); + auto rv = poll(pfds, 1, ms); if (rv < 0) { perror("Recieve poll"); - Close(fds->fd); + Close(pfds->fd); return true; } if (0 == rv) return false; // timeout - if (fds->revents & POLLIN) + if (pfds->revents & POLLIN) { - return receive(fds->fd, packet); + return receive(pfds->fd, packet); } - if (fds->revents & POLLERR || fds->revents & POLLHUP) + if (pfds->revents & POLLERR || pfds->revents & POLLHUP) { - std::cerr << ((fds->revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on fd " << fds->fd << ", closing socket" << std::endl; - Close(fds->fd); + std::cerr << ((pfds->revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on module " << module << "', closing socket" << std::endl; + Close(pfds->fd); return true; } @@ -183,16 +195,25 @@ bool CTCServer::Receive(char module, STCPacket &packet, int ms) bool CTCServer::Open(const std::string &address, const std::string &modules, uint16_t port) { m_Modules.assign(modules); - m_Pfd.resize(m_Modules.size()+1); + + m_Ip = CIp(address.c_str(), AF_UNSPEC, SOCK_STREAM, port); + + m_Pfd.resize(m_Modules.size()); for (auto &pf : m_Pfd) { pf.fd = -1; pf.events = POLLIN; + pf.revents = 0; } - CIp ip(address.c_str(), AF_UNSPEC, SOCK_STREAM, port); + std::cout << "Waiting for " << m_Modules.size() << " transcoder connection(s) on " << m_Ip << "..." << std::endl; - int fd = socket(ip.GetFamily(), SOCK_STREAM, 0); + return Accept(); +} + +bool CTCServer::Accept() +{ + auto fd = socket(m_Ip.GetFamily(), SOCK_STREAM, 0); if (fd < 0) { perror("Open socket"); @@ -200,7 +221,7 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin } int yes = 1; - int rv = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)); + auto rv = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)); if (rv < 0) { close(fd); @@ -208,7 +229,7 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin return true; } - rv = bind(fd, ip.GetCPointer(), ip.GetSize()); + rv = bind(fd, m_Ip.GetCPointer(), m_Ip.GetSize()); if (rv < 0) { close(fd); @@ -225,37 +246,19 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin return true; } - m_Pfd.back().fd = fd; - - std::cout << "Waiting for " << m_Modules.size() << " transcoder connection(s) on " << ip << "..." << std::endl; - return Accept(); -} - -bool CTCServer::Accept() -{ while (any_are_closed()) { if (acceptone()) return true; } + close(fd); + return false; } bool CTCServer::acceptone() { - auto rv = poll(&m_Pfd.back(), 1, 10); - if (rv < 0) - { - perror("Accept poll"); - return true; - } - - if (0 == rv) - return false; // timeout - - // rv has to be 1, so, here comes a connect request - CIp their_addr; // connector's address information socklen_t sin_size = sizeof(struct sockaddr_storage); @@ -268,7 +271,7 @@ bool CTCServer::acceptone() } char mod; - rv = recv(newfd, &mod, 1, 0); // block to get the identification byte + int rv = recv(newfd, &mod, 1, MSG_WAITALL); // block to get the identification byte if (rv != 1) { if (rv < 0) @@ -403,8 +406,10 @@ bool CTCClient::ReConnect() bool CTCClient::Receive(std::queue> &queue, int ms) { - const auto n = m_Pfd.size(); - auto rv = poll(m_Pfd.data(), n, ms); + for (auto &pfd : m_Pfd) + pfd.revents = 0; + + auto rv = poll(m_Pfd.data(), m_Pfd.size(), ms); if (rv < 0) { @@ -418,6 +423,9 @@ bool CTCClient::Receive(std::queue> &queue, int ms) bool some_closed = false; for (auto &pfd : m_Pfd) { + if (pfd.fd < 0) + continue; + if (pfd.revents & POLLIN) { auto p_tcpack = std::make_unique(); diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index cf1f269..cdbd9d1 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -24,6 +24,7 @@ #include #include +#include "IP.h" #include "TCPacketDef.h" class CTCSocket @@ -59,6 +60,7 @@ public: bool Accept(); private: + CIp m_Ip; bool any_are_closed(); bool acceptone(); }; @@ -73,7 +75,7 @@ public: bool ReConnect(); private: - bool Connect(char module); std::string m_Address; uint16_t m_Port; + bool Connect(char module); }; From a46e900121badd430c8844c29e0c3b9470ac6e37 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 17 May 2024 14:14:31 -0700 Subject: [PATCH 24/41] 2 TC Server bugs --- reflector/Reflector.cpp | 5 ++++- reflector/TCSocket.cpp | 6 +++--- reflector/TCSocket.h | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index 360a34c..1cfc3fc 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -70,7 +70,10 @@ bool CReflector::Start(void) // init transcoder comms if (port) - g_TCServer.Open(g_Configure.GetString(g_Keys.tc.bind), tcmods, port); + { + if (g_TCServer.Open(g_Configure.GetString(g_Keys.tc.bind), tcmods, port)) + return true; + } // init gate keeper. It can only return true! g_GateKeeper.Init(); diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 3699dae..3b80521 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -248,7 +248,7 @@ bool CTCServer::Accept() while (any_are_closed()) { - if (acceptone()) + if (acceptone(fd)) return true; } @@ -257,13 +257,13 @@ bool CTCServer::Accept() return false; } -bool CTCServer::acceptone() +bool CTCServer::acceptone(int fd) { CIp their_addr; // connector's address information socklen_t sin_size = sizeof(struct sockaddr_storage); - auto newfd = accept(m_Pfd.back().fd, their_addr.GetPointer(), &sin_size); + auto newfd = accept(fd, their_addr.GetPointer(), &sin_size); if (newfd < 0) { perror("Accept accept"); diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index cdbd9d1..fb66ae0 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -62,7 +62,7 @@ public: private: CIp m_Ip; bool any_are_closed(); - bool acceptone(); + bool acceptone(int fd); }; class CTCClient : public CTCSocket From f0486ec1f97570d5582a594e8a067e9ab2c9a9c2 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 17 May 2024 14:50:36 -0700 Subject: [PATCH 25/41] TC server Receive retval bug --- reflector/TCSocket.cpp | 25 ++++++++++++++----------- reflector/TCSocket.h | 3 ++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 3b80521..a950df3 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -149,47 +149,49 @@ bool CTCSocket::receive(int fd, STCPacket &packet) return false; } -// returns true on error +// returns false if there is data to return bool CTCServer::Receive(char module, STCPacket &packet, int ms) { + bool rv = false; const auto pos = m_Modules.find(module); if (pos == std::string::npos) { std::cerr << "Can't receive on unconfigured module '" << module << "'" << std::endl; - return true; + return rv; } auto pfds = &m_Pfd[pos]; if (pfds->fd < 0) { std::cerr << "Can't receive on module '" << module << "' because it's closed" << std::endl; - return true; + return rv; } - auto rv = poll(pfds, 1, ms); - if (rv < 0) + auto n = poll(pfds, 1, ms); + if (n < 0) { perror("Recieve poll"); Close(pfds->fd); - return true; + return rv; } - if (0 == rv) - return false; // timeout + if (0 == n) + return rv; // timeout if (pfds->revents & POLLIN) { - return receive(pfds->fd, packet); + rv = ! receive(pfds->fd, packet); } + // I think it's possible that we read the data, but the socket had an error after the read... + // So we'll check... if (pfds->revents & POLLERR || pfds->revents & POLLHUP) { std::cerr << ((pfds->revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on module " << module << "', closing socket" << std::endl; Close(pfds->fd); - return true; } - return false; + return rv; } bool CTCServer::Open(const std::string &address, const std::string &modules, uint16_t port) @@ -404,6 +406,7 @@ bool CTCClient::ReConnect() return rv; } +// returns true if there's an error, but there still make be something in the returned queue bool CTCClient::Receive(std::queue> &queue, int ms) { for (auto &pfd : m_Pfd) diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index fb66ae0..3a612d4 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -39,7 +39,7 @@ public: void Close(int fd); // close a specific file descriptor bool receive(int fd, STCPacket &packet); - // All bool functions return true if there was an error + // All bool functions, except Server Receive, return true if there was an error bool Send(const STCPacket *packet); int GetFD(char module) const; // can return -1! @@ -56,6 +56,7 @@ public: CTCServer() : CTCSocket() {} ~CTCServer() {} bool Open(const std::string &address, const std::string &modules, uint16_t port); + // Returns true if there is data bool Receive(char module, STCPacket &packet, int ms); bool Accept(); From 1a66c49dca8c3069828c1e70672eae605edb1236 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 17 May 2024 14:59:14 -0700 Subject: [PATCH 26/41] another bug, this time in CodecStream --- reflector/CodecStream.cpp | 2 +- reflector/TCSocket.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index a84f6aa..ba8457c 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -127,7 +127,7 @@ void CCodecStream::Task(void) } STCPacket pack; - if (! g_TCServer.Receive(m_CSModule, pack, 8)) + if (g_TCServer.Receive(m_CSModule, pack, 8)) { // update statistics double rt = pack.rt_timer.time(); // the round-trip time diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index a950df3..f16cb50 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -149,7 +149,7 @@ bool CTCSocket::receive(int fd, STCPacket &packet) return false; } -// returns false if there is data to return +// returns true if there is data to return bool CTCServer::Receive(char module, STCPacket &packet, int ms) { bool rv = false; @@ -183,7 +183,7 @@ bool CTCServer::Receive(char module, STCPacket &packet, int ms) rv = ! receive(pfds->fd, packet); } - // I think it's possible that we read the data, but the socket had an error after the read... + // I think it's possible that even if we read the data, the socket can have an error after the read... // So we'll check... if (pfds->revents & POLLERR || pfds->revents & POLLHUP) { From d4ad8a2e9ced8e96565aaa75639801c124c6e3e0 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Sat, 18 May 2024 10:16:24 -0700 Subject: [PATCH 27/41] RT Stats bug (pass pointers) --- reflector/CodecStream.cpp | 4 ++-- reflector/TCPacketDef.h | 4 ---- reflector/TCSocket.cpp | 12 ++++++------ reflector/TCSocket.h | 4 ++-- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index ba8457c..c53e851 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -127,10 +127,10 @@ void CCodecStream::Task(void) } STCPacket pack; - if (g_TCServer.Receive(m_CSModule, pack, 8)) + if (g_TCServer.Receive(m_CSModule, &pack, 8)) { // update statistics - double rt = pack.rt_timer.time(); // the round-trip time + auto rt = pack.rt_timer.time(); // the round-trip time if (0 == m_RTCount) { m_RTMin = rt; diff --git a/reflector/TCPacketDef.h b/reflector/TCPacketDef.h index 43abf5c..203da46 100644 --- a/reflector/TCPacketDef.h +++ b/reflector/TCPacketDef.h @@ -19,10 +19,6 @@ #include "Timer.h" -// unix socket names -#define TC2REF "TC2URFMod" -#define REF2TC "URF2TC" - enum class ECodecType : std::uint8_t { none = 0, dstar = 1, dmr = 2, c2_1600 = 3, c2_3200 = 4, p25 = 5, usrp = 6 }; using STCPacket = struct tcpacket_tag { diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index f16cb50..0c1bf75 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -127,9 +127,9 @@ bool CTCSocket::Send(const STCPacket *packet) return false; } -bool CTCSocket::receive(int fd, STCPacket &packet) +bool CTCSocket::receive(int fd, STCPacket *packet) { - auto n = recv(fd, &packet, sizeof(STCPacket), MSG_WAITALL); + auto n = recv(fd, packet, sizeof(STCPacket), MSG_WAITALL); if (n < 0) { perror("Receive recv"); @@ -139,18 +139,18 @@ bool CTCSocket::receive(int fd, STCPacket &packet) if (0 == n) { - std::cerr << "recv() returned zero bytes from mdule '" << GetMod(fd) << "'"; + std::cerr << "recv(): Module '" << GetMod(fd) << "' has been closed on the server" << std::endl; Close(fd); return true; } if (n != sizeof(STCPacket)) - std::cout << "WARNING: Receive only read " << n << " bytes of the transcoder packet from module '" << GetMod(fd) << "'" << std::endl; + std::cout << "receive() only read " << n << " bytes of the transcoder packet from module '" << GetMod(fd) << "'" << std::endl; return false; } // returns true if there is data to return -bool CTCServer::Receive(char module, STCPacket &packet, int ms) +bool CTCServer::Receive(char module, STCPacket *packet, int ms) { bool rv = false; const auto pos = m_Modules.find(module); @@ -432,7 +432,7 @@ bool CTCClient::Receive(std::queue> &queue, int ms) if (pfd.revents & POLLIN) { auto p_tcpack = std::make_unique(); - if (receive(pfd.fd, *p_tcpack)) + if (receive(pfd.fd, p_tcpack.get())) { p_tcpack.reset(); } diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 3a612d4..6ef5788 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -37,7 +37,7 @@ public: void Close(); // close all open sockets void Close(char module); // close a specific module void Close(int fd); // close a specific file descriptor - bool receive(int fd, STCPacket &packet); + bool receive(int fd, STCPacket *packet); // All bool functions, except Server Receive, return true if there was an error bool Send(const STCPacket *packet); @@ -57,7 +57,7 @@ public: ~CTCServer() {} bool Open(const std::string &address, const std::string &modules, uint16_t port); // Returns true if there is data - bool Receive(char module, STCPacket &packet, int ms); + bool Receive(char module, STCPacket *packet, int ms); bool Accept(); private: From 38e38824419819b42b49bf7b5dce99fdf9a514d9 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 20 May 2024 05:13:18 -0700 Subject: [PATCH 28/41] removed CTimer from STCPacket --- reflector/CodecStream.cpp | 36 ++++---- reflector/DVFramePacket.cpp | 175 +++++++++++++++++------------------- reflector/DVFramePacket.h | 11 ++- reflector/Packet.cpp | 123 ++++++++++++------------- reflector/Packet.h | 5 +- reflector/TCPacketDef.h | 3 - 6 files changed, 176 insertions(+), 177 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index c53e851..27643ba 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -129,23 +129,6 @@ void CCodecStream::Task(void) STCPacket pack; if (g_TCServer.Receive(m_CSModule, &pack, 8)) { - // update statistics - auto rt = pack.rt_timer.time(); // the round-trip time - if (0 == m_RTCount) - { - m_RTMin = rt; - m_RTMax = rt; - } - else - { - if (rt < m_RTMin) - m_RTMin = rt; - else if (rt > m_RTMax) - m_RTMax = rt; - } - m_RTSum += rt; - m_RTCount++; - if ( m_LocalQueue.IsEmpty() ) { std::cout << "Unexpected transcoded packet received from transcoder: Module='" << pack.module << "' StreamID=" << std::hex << std::showbase << ntohs(pack.streamid) << std::endl; @@ -158,6 +141,23 @@ void CCodecStream::Task(void) // make sure this is the correct packet if ((pack.streamid == Packet->GetCodecPacket()->streamid) && (pack.sequence == Packet->GetCodecPacket()->sequence)) { + // update statistics + auto rt =Packet->m_rtTimer.time(); // the round-trip time + if (0 == m_RTCount) + { + m_RTMin = rt; + m_RTMax = rt; + } + else + { + if (rt < m_RTMin) + m_RTMin = rt; + else if (rt > m_RTMax) + m_RTMax = rt; + } + m_RTSum += rt; + m_RTCount++; + // update content with transcoded data Packet->SetCodecData(&pack); // mark the DStar sync frames if the source isn't dstar @@ -207,6 +207,7 @@ void CCodecStream::Task(void) return; } + Frame->m_rtTimer.start(); // start the round-trip timer if (g_TCServer.Send(Frame->GetCodecPacket())) { // ditto, we'll try to fix this on the next pass @@ -214,6 +215,7 @@ void CCodecStream::Task(void) } // the fd was good and then the send was successful, so... // push the frame to our local queue where it can wait for the transcoder + m_LocalQueue.Push(std::move(m_Queue.Pop())); } } diff --git a/reflector/DVFramePacket.cpp b/reflector/DVFramePacket.cpp index 0dfc2b1..d00c00c 100644 --- a/reflector/DVFramePacket.cpp +++ b/reflector/DVFramePacket.cpp @@ -23,14 +23,14 @@ // default constructor CDvFramePacket::CDvFramePacket() { - memset(m_TCPack.dstar, 0, 9); - memset(m_uiDvData, 0, 3); - memset(m_TCPack.dmr, 0, 9); - memset(m_uiDvSync, 0, 7); - memset(m_TCPack.m17, 0, 16); - memset(m_TCPack.p25, 0, 11); - memset(m_TCPack.usrp, 0, 320); - memset(m_Nonce, 0, 14); + memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); + memset(m_uiDvData, 0, sizeof(m_uiDvData)); + memset(m_TCPack.dmr, 0, sizeof(m_TCPack.dmr)); + memset(m_uiDvSync, 0, sizeof(m_uiDvSync)); + memset(m_TCPack.m17, 0, sizeof(m_TCPack.m17)); + memset(m_TCPack.p25, 0, sizeof(m_TCPack.p25)); + memset(m_TCPack.usrp, 0, sizeof(m_TCPack.usrp)); + memset(m_Nonce, 0, sizeof(m_Nonce)); m_TCPack.codec_in = ECodecType::none; }; @@ -38,14 +38,14 @@ CDvFramePacket::CDvFramePacket() CDvFramePacket::CDvFramePacket(const SDStarFrame *dvframe, uint16_t sid, uint8_t pid) : CPacket(sid, pid) { - memcpy(m_TCPack.dstar, dvframe->AMBE, 9); - memcpy(m_uiDvData, dvframe->DVDATA, 3); - memset(m_TCPack.dmr, 0, 9); - memset(m_uiDvSync, 0, 7); - memset(m_TCPack.m17, 0, 16); - memset(m_TCPack.p25, 0, 11); - memset(m_TCPack.usrp, 0, 320); - memset(m_Nonce, 0, 14); + memcpy(m_TCPack.dstar, dvframe->AMBE, sizeof(m_TCPack.dstar)); + memcpy(m_uiDvData, dvframe->DVDATA, sizeof(m_uiDvData)); + memset(m_TCPack.dmr, 0, sizeof(m_TCPack.dmr)); + memset(m_uiDvSync, 0, sizeof(m_uiDvSync)); + memset(m_TCPack.m17, 0, sizeof(m_TCPack.m17)); + memset(m_TCPack.p25, 0, sizeof(m_TCPack.p25)); + memset(m_TCPack.usrp, 0, sizeof(m_TCPack.usrp)); + memset(m_Nonce, 0, sizeof(m_Nonce)); m_TCPack.codec_in = ECodecType::dstar; } @@ -53,14 +53,14 @@ CDvFramePacket::CDvFramePacket(const SDStarFrame *dvframe, uint16_t sid, uint8_t CDvFramePacket::CDvFramePacket(const uint8_t *ambe, const uint8_t *sync, uint16_t sid, uint8_t pid, uint8_t spid, bool islast) : CPacket(sid, pid, spid, islast) { - memcpy(m_TCPack.dmr, ambe, 9); - memcpy(m_uiDvSync, sync, 7); - memset(m_TCPack.dstar, 0, 9); - memset(m_uiDvData, 0, 3); - memset(m_TCPack.m17, 0, 16); - memset(m_TCPack.p25, 0, 11); - memset(m_TCPack.usrp, 0, 320); - memset(m_Nonce, 0, 14); + memcpy(m_TCPack.dmr, ambe, sizeof(m_TCPack.dmr)); + memcpy(m_uiDvSync, sync, sizeof(m_uiDvSync)); + memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); + memset(m_uiDvData, 0, sizeof(m_uiDvData)); + memset(m_TCPack.m17, 0, sizeof(m_TCPack.m17)); + memset(m_TCPack.p25, 0, sizeof(m_TCPack.p25)); + memset(m_TCPack.usrp, 0, sizeof(m_TCPack.usrp)); + memset(m_Nonce, 0, sizeof(m_Nonce)); m_TCPack.codec_in = ECodecType::dmr; } @@ -68,14 +68,14 @@ CDvFramePacket::CDvFramePacket(const uint8_t *ambe, const uint8_t *sync, uint16_ CDvFramePacket::CDvFramePacket(const uint8_t *ambe, uint16_t sid, uint8_t pid, uint8_t spid, uint8_t fid, CCallsign cs, bool islast) : CPacket(sid, pid, spid, fid, islast) { - memcpy(m_TCPack.dmr, ambe, 9); - memset(m_uiDvSync, 0, 7); - memset(m_TCPack.dstar, 0, 9); - memset(m_uiDvData, 0, 3); - memset(m_TCPack.m17, 0, 16); - memset(m_TCPack.p25, 0, 11); - memset(m_TCPack.usrp, 0, 320); - memset(m_Nonce, 0, 14); + memcpy(m_TCPack.dmr, ambe, sizeof(m_TCPack.dmr)); + memset(m_uiDvSync, 0, sizeof(m_uiDvSync)); + memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); + memset(m_uiDvData, 0, sizeof(m_uiDvData)); + memset(m_TCPack.m17, 0, sizeof(m_TCPack.m17)); + memset(m_TCPack.p25, 0, sizeof(m_TCPack.p25)); + memset(m_TCPack.usrp, 0, sizeof(m_TCPack.usrp)); + memset(m_Nonce, 0, sizeof(m_Nonce)); m_TCPack.codec_in = ECodecType::dmr; uint8_t c[12]; cs.GetCallsign(c); @@ -87,10 +87,10 @@ CDvFramePacket::CDvFramePacket (uint16_t sid, uint8_t dstarpid, const uint8_t *dstarambe, const uint8_t *dstardvdata, uint8_t dmrpid, uint8_t dprspid, const uint8_t *dmrambe, const uint8_t *dmrsync, ECodecType codecInType, bool islast) : CPacket(sid, dstarpid, dmrpid, dprspid, 0xFF, 0xFF, 0xFF, codecInType, islast) { - ::memcpy(m_TCPack.dstar, dstarambe, 9); - ::memcpy(m_uiDvData, dstardvdata, 3); - ::memcpy(m_TCPack.dmr, dmrambe, 9); - ::memcpy(m_uiDvSync, dmrsync, 7); + memcpy(m_TCPack.dstar, dstarambe, sizeof(m_TCPack.dstar)); + memcpy(m_TCPack.dmr, dmrambe, sizeof(m_TCPack.dmr)); + memcpy(m_uiDvData, dstardvdata, sizeof(m_uiDvData)); + memcpy(m_uiDvSync, dmrsync, sizeof(m_uiDvSync)); m_TCPack.codec_in = codecInType; } @@ -98,14 +98,14 @@ CDvFramePacket::CDvFramePacket CDvFramePacket::CDvFramePacket(const CM17Packet &m17) : CPacket(m17) { - memset(m_TCPack.dstar, 0, 9); - memset(m_uiDvData, 0, 3); - memset(m_TCPack.dmr, 0, 9); - memset(m_uiDvSync, 0, 7); - memcpy(m_TCPack.m17, m17.GetPayload(), 16); - memcpy(m_Nonce, m17.GetNonce(), 14); - memset(m_TCPack.p25, 0, 11); - memset(m_TCPack.usrp, 0, 320); + memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); + memset(m_TCPack.dmr, 0, sizeof(m_TCPack.dmr)); + memset(m_TCPack.p25, 0, sizeof(m_TCPack.p25)); + memset(m_TCPack.usrp, 0, sizeof(m_TCPack.usrp)); + memset(m_uiDvData, 0, sizeof(m_uiDvData)); + memset(m_uiDvSync, 0, sizeof(m_uiDvSync)); + memcpy(m_TCPack.m17, m17.GetPayload(), sizeof(m_TCPack.m17)); + memcpy(m_Nonce, m17.GetNonce(), sizeof(m_Nonce)); switch (0x6U & m17.GetFrameType()) { case 0x4U: @@ -124,14 +124,14 @@ CDvFramePacket::CDvFramePacket(const CM17Packet &m17) : CPacket(m17) CDvFramePacket::CDvFramePacket(const uint8_t *imbe, uint16_t streamid, bool islast) : CPacket(streamid, false, islast) { - memcpy(m_TCPack.p25, imbe, 11); - memset(m_TCPack.dmr, 0, 9); - memset(m_uiDvSync, 0, 7); - memset(m_TCPack.dstar, 0, 9); - memset(m_uiDvData, 0, 3); - memset(m_TCPack.m17, 0, 16); - memset(m_TCPack.usrp, 0, 320); - memset(m_Nonce, 0, 14); + memcpy(m_TCPack.p25, imbe, sizeof(m_TCPack.p25)); + memset(m_TCPack.dmr, 0, sizeof(m_TCPack.dmr)); + memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); + memset(m_TCPack.m17, 0, sizeof(m_TCPack.m17)); + memset(m_TCPack.usrp, 0, sizeof(m_TCPack.usrp)); + memset(m_uiDvSync, 0, sizeof(m_uiDvSync)); + memset(m_uiDvData, 0, sizeof(m_uiDvData)); + memset(m_Nonce, 0, sizeof(m_Nonce)); m_TCPack.codec_in = ECodecType::p25; } @@ -139,14 +139,14 @@ CDvFramePacket::CDvFramePacket(const uint8_t *imbe, uint16_t streamid, bool isla CDvFramePacket::CDvFramePacket(const uint8_t *ambe, uint16_t sid, uint8_t pid, bool islast) : CPacket(sid, pid, islast) { - memcpy(m_TCPack.dmr, ambe, 9); - memset(m_uiDvSync, 0, 7); - memset(m_TCPack.dstar, 0, 9); - memset(m_uiDvData, 0, 3); - memset(m_TCPack.m17, 0, 16); - memset(m_TCPack.p25, 0, 11); - memset(m_TCPack.usrp, 0, 320); - memset(m_Nonce, 0, 14); + memcpy(m_TCPack.dmr, ambe, sizeof(m_TCPack.dmr)); + memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); + memset(m_TCPack.m17, 0, sizeof(m_TCPack.m17)); + memset(m_TCPack.p25, 0, sizeof(m_TCPack.p25)); + memset(m_TCPack.usrp, 0, sizeof(m_TCPack.usrp)); + memset(m_uiDvSync, 0, sizeof(m_uiDvSync)); + memset(m_uiDvData, 0, sizeof(m_uiDvData)); + memset(m_Nonce, 0, sizeof(m_Nonce)); m_TCPack.codec_in = ECodecType::dmr; } @@ -157,13 +157,13 @@ CDvFramePacket::CDvFramePacket(const int16_t *usrp, uint16_t streamid, bool isla for(int i = 0; i < 160; ++i){ m_TCPack.usrp[i] = usrp[i]; } - memset(m_TCPack.p25, 0, 11); - memset(m_TCPack.dmr, 0, 9); - memset(m_uiDvSync, 0, 7); - memset(m_TCPack.dstar, 0, 9); - memset(m_uiDvData, 0, 3); - memset(m_TCPack.m17, 0, 16); - memset(m_Nonce, 0, 14); + memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); + memset(m_TCPack.dmr, 0, sizeof(m_TCPack.dmr)); + memset(m_TCPack.m17, 0, sizeof(m_TCPack.m17)); + memset(m_TCPack.p25, 0, sizeof(m_TCPack.p25)); + memset(m_uiDvSync, 0, sizeof(m_uiDvSync)); + memset(m_uiDvData, 0, sizeof(m_uiDvData)); + memset(m_Nonce, 0, sizeof(m_Nonce)); m_TCPack.codec_in = ECodecType::usrp; } @@ -172,12 +172,6 @@ std::unique_ptr CDvFramePacket::Copy(void) return std::unique_ptr(new CDvFramePacket(*this)); } -// Network -unsigned int CDvFramePacket::GetNetworkSize() -{ - return CPacket::GetNetworkSize() + 4 + 3 + 7 + 14 + 9 + 9 + 16 + 11 + 320; -} - CDvFramePacket::CDvFramePacket(const CBuffer &buf) : CPacket(buf) { if (buf.size() >= GetNetworkSize()) @@ -188,14 +182,14 @@ CDvFramePacket::CDvFramePacket(const CBuffer &buf) : CPacket(buf) for (unsigned int i=0; i<4; i++) seq = 0x100u * seq + data[off+i]; off += 4; - memcpy(m_uiDvData, data+off, 3); off += 3; - memcpy(m_uiDvSync, data+off, 7); off += 7; - memcpy(m_Nonce, data+off, 14); off += 14; - memcpy(m_TCPack.dstar, data+off, 9); off += 9; - memcpy(m_TCPack.dmr, data+off, 9); off += 9; - memcpy(m_TCPack.m17, data+off, 16); off += 16; - memcpy(m_TCPack.p25, data+off, 11); off += 11; - memcpy(m_TCPack.usrp, data+off, 320); + memcpy(m_uiDvData, data+off, sizeof(m_uiDvData)); off += sizeof(m_uiDvData); + memcpy(m_uiDvSync, data+off, sizeof(m_uiDvSync)); off += sizeof(m_uiDvSync); + memcpy(m_Nonce, data+off, sizeof(m_Nonce)); off += sizeof(m_Nonce); + memcpy(m_TCPack.dstar, data+off, sizeof(m_TCPack.dstar)); off += sizeof(m_TCPack.dstar); + memcpy(m_TCPack.dmr, data+off, sizeof(m_TCPack.dmr)); off += sizeof (m_TCPack.dmr); + memcpy(m_TCPack.m17, data+off, sizeof(m_TCPack.m17)); off += sizeof(m_TCPack.m17); + memcpy(m_TCPack.p25, data+off, sizeof(m_TCPack.p25)); off += sizeof(m_TCPack.p25); + memcpy(m_TCPack.usrp, data+off, sizeof(m_TCPack.usrp)); SetTCParams(seq); } else @@ -212,14 +206,14 @@ void CDvFramePacket::EncodeInterlinkPacket(CBuffer &buf) const data[off++] = (m_TCPack.sequence >> 16) & 0xffu; data[off++] = (m_TCPack.sequence >> 8) & 0xffu; data[off++] = m_TCPack.sequence & 0xffu; - memcpy(data+off, m_uiDvData, 3); off += 3; - memcpy(data+off, m_uiDvSync, 7); off += 7; - memcpy(data+off, m_Nonce, 14); off += 14; - memcpy(data+off, m_TCPack.dstar, 9); off += 9; - memcpy(data+off, m_TCPack.dmr, 9); off += 9; - memcpy(data+off, m_TCPack.m17, 16); off += 16; - memcpy(data+off, m_TCPack.p25, 11); off += 11; - memcpy(data+off, m_TCPack.usrp, 320); + memcpy(data+off, m_uiDvData, sizeof(m_uiDvData)); off += sizeof(m_uiDvData); + memcpy(data+off, m_uiDvSync, sizeof(m_uiDvSync)); off += sizeof(m_uiDvSync); + memcpy(data+off, m_Nonce, sizeof(m_Nonce)); off += sizeof(m_Nonce); + memcpy(data+off, m_TCPack.dstar, sizeof(m_TCPack.dstar)); off += sizeof(m_TCPack.dstar); + memcpy(data+off, m_TCPack.dmr, sizeof(m_TCPack.dmr)); off += sizeof(m_TCPack.dmr); + memcpy(data+off, m_TCPack.m17, sizeof(m_TCPack.m17)); off += sizeof(m_TCPack.m17); + memcpy(data+off, m_TCPack.p25, sizeof(m_TCPack.p25)); off += sizeof(m_TCPack.p25); + memcpy(data+off, m_TCPack.usrp, sizeof(m_TCPack.usrp)); } //////////////////////////////////////////////////////////////////////////////////////// @@ -250,7 +244,7 @@ const uint8_t *CDvFramePacket::GetCodecData(ECodecType type) const void CDvFramePacket::SetDvData(const uint8_t *DvData) { - memcpy(m_uiDvData, DvData, 3); + memcpy(m_uiDvData, DvData, sizeof(m_uiDvData)); } void CDvFramePacket::SetCodecData(const STCPacket *pack) @@ -264,5 +258,4 @@ void CDvFramePacket::SetTCParams(uint32_t seq) m_TCPack.streamid = m_uiStreamId; m_TCPack.is_last = m_bLastPacket; m_TCPack.module = m_cModule; - m_TCPack.rt_timer.start(); } diff --git a/reflector/DVFramePacket.h b/reflector/DVFramePacket.h index d4c964a..33248cd 100644 --- a/reflector/DVFramePacket.h +++ b/reflector/DVFramePacket.h @@ -20,6 +20,7 @@ #include "Packet.h" #include "Callsign.h" +#include "Timer.h" //////////////////////////////////////////////////////////////////////////////////////// // defines @@ -59,7 +60,12 @@ public: CDvFramePacket(const int16_t *usrp, uint16_t streamid, bool islast); // URF Network CDvFramePacket(const CBuffer &buf); - static unsigned int GetNetworkSize(); + + static constexpr unsigned GetNetworkSize() noexcept + { + return CPacket::GetNetworkSize() + sizeof(m_uiDvData) + sizeof(m_uiDvSync) + sizeof(m_Nonce) + sizeof(m_TCPack.dstar) + sizeof (m_TCPack.dmr) + sizeof(m_TCPack.m17) + sizeof(m_TCPack.p25) + sizeof(m_TCPack.usrp) + sizeof(m_TCPack.sequence); + } + void EncodeInterlinkPacket(CBuffer &buf) const; // identity @@ -80,6 +86,9 @@ public: void SetCodecData(const STCPacket *pack); void SetTCParams(uint32_t seq); + // the round-trip timer + CTimer m_rtTimer; + protected: // data (dstar) uint8_t m_uiDvData[3]; diff --git a/reflector/Packet.cpp b/reflector/Packet.cpp index 2700369..50bba1f 100644 --- a/reflector/Packet.cpp +++ b/reflector/Packet.cpp @@ -38,11 +38,6 @@ CPacket::CPacket() }; // for the network -unsigned int CPacket::GetNetworkSize() -{ - return 20u; -} - CPacket::CPacket(const CBuffer &buf) { if (buf.size() > 19) @@ -53,7 +48,7 @@ CPacket::CPacket(const CBuffer &buf) m_bLastPacket = data[6] ? true : false; m_cModule = data[7]; m_uiStreamId = data[8]*0x100u + data[9]; - m_uiM17FrameNumber = data[10]*0x1000000u + data[11]*0x10000u + data[12]*0x100 + data[13]; + m_uiM17FrameNumber = data[10]*0x1000000u + data[11]*0x10000u + data[12]*0x100u + data[13]; m_uiDstarPacketId = data[14]; m_uiDmrPacketId = data[15]; m_uiDmrPacketSubid = data[16]; @@ -93,17 +88,17 @@ CPacket::CPacket(uint16_t sid, uint8_t dstarpid) { m_uiStreamId = sid; m_uiDstarPacketId = dstarpid; - m_uiDmrPacketId = 0xFF; - m_uiDmrPacketSubid = 0xFF; - m_uiYsfPacketId = 0xFF; - m_uiYsfPacketSubId = 0xFF; - m_uiYsfPacketFrameId = 0xFF; - m_uiNXDNPacketId = 0xFF; - m_uiM17FrameNumber = 0xFFFFFFFFU; + m_uiDmrPacketId = 0xffu; + m_uiDmrPacketSubid = 0xffu; + m_uiYsfPacketId = 0xffu; + m_uiYsfPacketSubId = 0xffu; + m_uiYsfPacketFrameId = 0xffu; + m_uiNXDNPacketId = 0xffu; + m_uiM17FrameNumber = 0xffffffffu; m_cModule = ' '; m_eOrigin = EOrigin::local; m_eCodecIn = ECodecType::dstar; - m_bLastPacket = (0x40U == (dstarpid & 0x40U)); + m_bLastPacket = (0x40u == (dstarpid & 0x40u)); }; // dmr constructor @@ -112,12 +107,12 @@ CPacket::CPacket(uint16_t sid, uint8_t dmrpid, uint8_t dmrspid, bool lastpacket) m_uiStreamId = sid; m_uiDmrPacketId = dmrpid; m_uiDmrPacketSubid = dmrspid; - m_uiDstarPacketId = 0xFF; - m_uiYsfPacketId = 0xFF; - m_uiYsfPacketSubId = 0xFF; - m_uiYsfPacketFrameId = 0xFF; - m_uiNXDNPacketId = 0xFF; - m_uiM17FrameNumber = 0xFFFFFFFFU; + m_uiDstarPacketId = 0xffu; + m_uiYsfPacketId = 0xffu; + m_uiYsfPacketSubId = 0xffu; + m_uiYsfPacketFrameId = 0xffu; + m_uiNXDNPacketId = 0xffu; + m_uiM17FrameNumber = 0xffffffffu; m_cModule = ' '; m_eOrigin = EOrigin::local; m_eCodecIn = ECodecType::dmr; @@ -131,11 +126,11 @@ CPacket::CPacket(uint16_t sid, uint8_t ysfpid, uint8_t ysfsubpid, uint8_t ysffri m_uiYsfPacketId = ysfpid; m_uiYsfPacketSubId = ysfsubpid; m_uiYsfPacketFrameId = ysffrid; - m_uiDstarPacketId = 0xFF; - m_uiDmrPacketId = 0xFF; - m_uiDmrPacketSubid = 0xFF; - m_uiNXDNPacketId = 0xFF; - m_uiM17FrameNumber = 0xFFFFFFFFU; + m_uiDstarPacketId = 0xffu; + m_uiDmrPacketId = 0xffu; + m_uiDmrPacketSubid = 0xffu; + m_uiNXDNPacketId = 0xffu; + m_uiM17FrameNumber = 0xffffffffu; m_cModule = ' '; m_eOrigin = EOrigin::local; m_eCodecIn = ECodecType::dmr; @@ -147,13 +142,13 @@ CPacket::CPacket(uint16_t sid, uint8_t pid, bool lastpacket) { m_uiStreamId = sid; m_uiNXDNPacketId = pid; - m_uiDmrPacketId = 0xFF; - m_uiDmrPacketSubid = 0xFF; - m_uiDstarPacketId = 0xFF; - m_uiYsfPacketId = 0xFF; - m_uiYsfPacketSubId = 0xFF; - m_uiYsfPacketFrameId = 0xFF; - m_uiM17FrameNumber = 0xFFFFFFFFU; + m_uiDmrPacketId = 0xffu; + m_uiDmrPacketSubid = 0xffu; + m_uiDstarPacketId = 0xffu; + m_uiYsfPacketId = 0xffu; + m_uiYsfPacketSubId = 0xffu; + m_uiYsfPacketFrameId = 0xffu; + m_uiM17FrameNumber = 0xffffffffu; m_cModule = ' '; m_eOrigin = EOrigin::local; m_eCodecIn = ECodecType::dmr; @@ -164,14 +159,14 @@ CPacket::CPacket(uint16_t sid, uint8_t pid, bool lastpacket) CPacket::CPacket(uint16_t sid, bool isusrp, bool lastpacket) { m_uiStreamId = sid; - m_uiDstarPacketId = 0xFF; - m_uiDmrPacketId = 0xFF; - m_uiDmrPacketSubid = 0xFF; - m_uiYsfPacketId = 0xFF; - m_uiYsfPacketSubId = 0xFF; - m_uiYsfPacketFrameId = 0xFF; - m_uiNXDNPacketId = 0xFF; - m_uiM17FrameNumber = 0xFFFFFFFFU; + m_uiDstarPacketId = 0xffu; + m_uiDmrPacketId = 0xffu; + m_uiDmrPacketSubid = 0xffu; + m_uiYsfPacketId = 0xffu; + m_uiYsfPacketSubId = 0xffu; + m_uiYsfPacketFrameId = 0xffu; + m_uiNXDNPacketId = 0xffu; + m_uiM17FrameNumber = 0xffffffffu; m_cModule = ' '; m_eOrigin = EOrigin::local; isusrp ? m_eCodecIn = ECodecType::usrp : ECodecType::p25; @@ -188,8 +183,8 @@ CPacket::CPacket(uint16_t sid, uint8_t dstarpid, uint8_t dmrpid, uint8_t dmrsubp m_uiYsfPacketId = ysfpid; m_uiYsfPacketSubId = ysfsubpid; m_uiYsfPacketFrameId = ysffrid; - m_uiM17FrameNumber = 0xFFFFFFFFU; - m_uiNXDNPacketId = 0xFF; + m_uiM17FrameNumber = 0xffffffffu; + m_uiNXDNPacketId = 0xffu; m_cModule = ' '; m_eOrigin = EOrigin::local; m_eCodecIn = codecIn; @@ -200,15 +195,15 @@ CPacket::CPacket(uint16_t sid, uint8_t dstarpid, uint8_t dmrpid, uint8_t dmrsubp CPacket::CPacket(const CM17Packet &m17) : CPacket() { m_uiStreamId = m17.GetStreamId(); - m_uiDstarPacketId = 0xFF; - m_uiDmrPacketId = 0xFF; - m_uiDmrPacketSubid = 0xFF; - m_uiYsfPacketId = 0xFF; - m_uiYsfPacketSubId = 0xFF; - m_uiYsfPacketFrameId = 0xFF; - m_uiNXDNPacketId = 0xFF; - m_eCodecIn = (0x6U == (0x6U & m17.GetFrameType())) ? ECodecType::c2_1600 : ECodecType::c2_3200; - m_uiM17FrameNumber = 0xFFFFU & m17.GetFrameNumber(); + m_uiDstarPacketId = 0xffu; + m_uiDmrPacketId = 0xffu; + m_uiDmrPacketSubid = 0xffu; + m_uiYsfPacketId = 0xffu; + m_uiYsfPacketSubId = 0xffu; + m_uiYsfPacketFrameId = 0xffu; + m_uiNXDNPacketId = 0xffu; + m_eCodecIn = (0x6u == (0x6u & m17.GetFrameType())) ? ECodecType::c2_1600 : ECodecType::c2_3200; + m_uiM17FrameNumber = 0xffffu & m17.GetFrameNumber(); m_bLastPacket = m17.IsLastPacket(); } @@ -223,31 +218,31 @@ void CPacket::UpdatePids(const uint32_t pid) // derived from each other // dstar pid needs update ? - if ( m_uiDstarPacketId == 0xFF ) + if ( m_uiDstarPacketId == 0xffu ) { - m_uiDstarPacketId = (pid % 21); + m_uiDstarPacketId = (pid % 21u); } // dmr pids need update ? - if ( m_uiDmrPacketId == 0xFF ) + if ( m_uiDmrPacketId == 0xffu ) { - m_uiDmrPacketId = ((pid / 3) % 6); - m_uiDmrPacketSubid = ((pid % 3) + 1); + m_uiDmrPacketId = ((pid / 3u) % 6u); + m_uiDmrPacketSubid = ((pid % 3u) + 1u); } // ysf pids need update ? - if ( m_uiYsfPacketId == 0xFF ) + if ( m_uiYsfPacketId == 0xffu ) { - m_uiYsfPacketId = ((pid / 5) % 8); - m_uiYsfPacketSubId = pid % 5; - m_uiYsfPacketFrameId = ((pid / 5) & 0x7FU) << 1; + m_uiYsfPacketId = ((pid / 5u) % 8u); + m_uiYsfPacketSubId = pid % 5u; + m_uiYsfPacketFrameId = ((pid / 5u) & 0x7fu) << 1; } - if ( m_uiNXDNPacketId == 0xFF ) + if ( m_uiNXDNPacketId == 0xffu ) { - m_uiNXDNPacketId = pid % 4; + m_uiNXDNPacketId = pid % 4u; } // m17 needs update? - if (m_uiM17FrameNumber == 0xFFFFFFFFU) + if (m_uiM17FrameNumber == 0xffffffffu) { // frames are every 20 milliseconds, so the m17 data will come every 40 milliseconds - m_uiM17FrameNumber = (pid / 2) % 0x8000U; + m_uiM17FrameNumber = (pid / 2) % 0x8000u; } } diff --git a/reflector/Packet.h b/reflector/Packet.h index 0ee7012..1082522 100644 --- a/reflector/Packet.h +++ b/reflector/Packet.h @@ -72,7 +72,10 @@ public: protected: // network void EncodeInterlinkPacket(const char *magic, CBuffer &Buffer) const; - static unsigned int GetNetworkSize(); + static constexpr unsigned GetNetworkSize() noexcept + { + return 4u + sizeof(ECodecType) + sizeof(EOrigin) + sizeof(bool) + sizeof(char) + sizeof(uint16_t) + sizeof(uint32_t) + 7u * sizeof(uint8_t); + } // data // if you change something here, you'll need to update the CBuffer ctor and EncodeInterlinkPacket()!!! diff --git a/reflector/TCPacketDef.h b/reflector/TCPacketDef.h index 203da46..a3681b7 100644 --- a/reflector/TCPacketDef.h +++ b/reflector/TCPacketDef.h @@ -17,12 +17,9 @@ #include -#include "Timer.h" - enum class ECodecType : std::uint8_t { none = 0, dstar = 1, dmr = 2, c2_1600 = 3, c2_3200 = 4, p25 = 5, usrp = 6 }; using STCPacket = struct tcpacket_tag { - CTimer rt_timer; uint32_t sequence; char module; bool is_last; From a726c9ee22c7b0c1300d488532b2a3f08443025a Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 20 May 2024 10:56:53 -0700 Subject: [PATCH 29/41] close fd if there is a client receive error --- reflector/DVFramePacket.cpp | 2 +- reflector/TCSocket.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/reflector/DVFramePacket.cpp b/reflector/DVFramePacket.cpp index d00c00c..fa1ff78 100644 --- a/reflector/DVFramePacket.cpp +++ b/reflector/DVFramePacket.cpp @@ -21,7 +21,7 @@ #include "DVFramePacket.h" // default constructor -CDvFramePacket::CDvFramePacket() +CDvFramePacket::CDvFramePacket() : CPacket() { memset(m_TCPack.dstar, 0, sizeof(m_TCPack.dstar)); memset(m_uiDvData, 0, sizeof(m_uiDvData)); diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 0c1bf75..f8a8260 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -139,7 +139,7 @@ bool CTCSocket::receive(int fd, STCPacket *packet) if (0 == n) { - std::cerr << "recv(): Module '" << GetMod(fd) << "' has been closed on the server" << std::endl; + std::cerr << "recv(): Module '" << GetMod(fd) << "' has been closed from the other side" << std::endl; Close(fd); return true; } @@ -435,6 +435,7 @@ bool CTCClient::Receive(std::queue> &queue, int ms) if (receive(pfd.fd, p_tcpack.get())) { p_tcpack.reset(); + Close(pfd.fd); } else { From ee1bd2228b3ffc49cec10b34a31e5069f3f14b64 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 20 May 2024 14:41:46 -0700 Subject: [PATCH 30/41] recovering from a closed TCP connection --- reflector/TCSocket.cpp | 41 +++++++++++++++++++++++++++++------------ reflector/TCSocket.h | 2 +- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index f8a8260..c818e45 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -44,12 +44,22 @@ void CTCSocket::Close(char mod) std::cerr << "Could not find module '" << mod << "'" << std::endl; return; } + if (m_Pfd[pos].fd < 0) + { + std::cerr << "Close(" << mod << ") is already closed" << std::endl; + return; + } Close(m_Pfd[pos].fd); m_Pfd[pos].fd = -1; } void CTCSocket::Close(int fd) { + if (fd < 0) + { + std::cerr << "Close(fd) : fd is -1" << std::endl; + return; + } for (auto &p : m_Pfd) { if (fd == p.fd) @@ -140,7 +150,6 @@ bool CTCSocket::receive(int fd, STCPacket *packet) if (0 == n) { std::cerr << "recv(): Module '" << GetMod(fd) << "' has been closed from the other side" << std::endl; - Close(fd); return true; } @@ -180,18 +189,25 @@ bool CTCServer::Receive(char module, STCPacket *packet, int ms) if (pfds->revents & POLLIN) { - rv = ! receive(pfds->fd, packet); + rv = receive(pfds->fd, packet); } - // I think it's possible that even if we read the data, the socket can have an error after the read... + // It's possible that even if we read the data, the socket can have an error after the read... // So we'll check... if (pfds->revents & POLLERR || pfds->revents & POLLHUP) { - std::cerr << ((pfds->revents & POLLERR) ? "POLLERR" : "POLLHUP") << " received on module " << module << "', closing socket" << std::endl; + if (pfds->revents & POLLERR) + std::cerr << "POLLERR received on module '" << module << "', closing socket" << std::endl; + if (pfds->revents & POLLHUP) + std::cerr << "POLLHUP received on module '" << module << "', closing socket" << std::endl; Close(pfds->fd); } + if (pfds->revents & POLLNVAL) + { + std::cerr << "POLLNVAL received on module " << module << "'" << std::endl; + } - return rv; + return ! rv; } bool CTCServer::Open(const std::string &address, const std::string &modules, uint16_t port) @@ -406,8 +422,7 @@ bool CTCClient::ReConnect() return rv; } -// returns true if there's an error, but there still make be something in the returned queue -bool CTCClient::Receive(std::queue> &queue, int ms) +void CTCClient::Receive(std::queue> &queue, int ms) { for (auto &pfd : m_Pfd) pfd.revents = 0; @@ -417,13 +432,12 @@ bool CTCClient::Receive(std::queue> &queue, int ms) if (rv < 0) { perror("Receive poll"); - return true; + return; } if (0 == rv) - return false; + return; - bool some_closed = false; for (auto &pfd : m_Pfd) { if (pfd.fd < 0) @@ -447,8 +461,11 @@ bool CTCClient::Receive(std::queue> &queue, int ms) { std::cerr << "IO ERROR on Receive module " << GetMod(pfd.fd) << std::endl; Close(pfd.fd); - some_closed = true; + } + if (pfd.revents & POLLNVAL) + { + std::cerr << "POLLNVAL received on fd " << pfd.fd << ", resetting to -1" << std::endl; + pfd.fd = -1; } } - return some_closed; } diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 6ef5788..8a2de8a 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -72,7 +72,7 @@ public: CTCClient() : CTCSocket(), m_Port(0) {} ~CTCClient() {} bool Open(const std::string &address, const std::string &modules, uint16_t port); - bool Receive(std::queue> &queue, int ms); + void Receive(std::queue> &queue, int ms); bool ReConnect(); private: From d58f5a24861c7d425a88c6d3d8add24f0db031d3 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 20 May 2024 15:03:15 -0700 Subject: [PATCH 31/41] TCPServer Receive bug fix --- reflector/TCSocket.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index c818e45..24c7658 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -57,7 +57,6 @@ void CTCSocket::Close(int fd) { if (fd < 0) { - std::cerr << "Close(fd) : fd is -1" << std::endl; return; } for (auto &p : m_Pfd) @@ -207,6 +206,8 @@ bool CTCServer::Receive(char module, STCPacket *packet, int ms) std::cerr << "POLLNVAL received on module " << module << "'" << std::endl; } + if (rv) + Close(pfds->fd); return ! rv; } From f9e67ce7e619ad0874985f91be656fad5d333c85 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 20 May 2024 15:44:16 -0700 Subject: [PATCH 32/41] use abort() --- reflector/CodecStream.cpp | 2 +- reflector/SafePacketQueue.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index 27643ba..27ae286 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -119,7 +119,7 @@ void CCodecStream::Task(void) if (g_TCServer.Accept()) // try to get a connection { std::cerr << "Unrecoverable ERROR! Quiting..." << std::endl; - exit(1); + abort(); } // Either Accept timed out, or it's possile that other Transcoder ports were instead reopened // So we'll check to see if this module is now open diff --git a/reflector/SafePacketQueue.h b/reflector/SafePacketQueue.h index 6c5635e..f1a8bf5 100644 --- a/reflector/SafePacketQueue.h +++ b/reflector/SafePacketQueue.h @@ -51,7 +51,7 @@ public: if (q.empty()) { std::cerr << "ERROR: CSavePacketQueue::Front() called, but queue is EMPTY!" << std::endl; - exit(1); + abort(); } return q.front(); } From 9ba2c8fefd0398e3400053dec9ae75158edb0786 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Wed, 22 May 2024 05:22:58 -0700 Subject: [PATCH 33/41] move TC server Accept to the reflector maintenance thread --- reflector/CodecStream.cpp | 18 ----------- reflector/Reflector.cpp | 64 ++++++++++++++++++++++----------------- reflector/Reflector.h | 4 +-- reflector/TCSocket.cpp | 24 ++++++++++----- reflector/TCSocket.h | 2 +- 5 files changed, 56 insertions(+), 56 deletions(-) diff --git a/reflector/CodecStream.cpp b/reflector/CodecStream.cpp index 27ae286..30bc9fa 100644 --- a/reflector/CodecStream.cpp +++ b/reflector/CodecStream.cpp @@ -108,24 +108,6 @@ void CCodecStream::Thread() void CCodecStream::Task(void) { - int fd = g_TCServer.GetFD(m_CSModule); - - // if the fd is not good we need to reestablish it - if (fd < 0) // log the situation - std::cout << "Lost connection to transcoder, module '" << m_CSModule << "', waiting for new connection..." << std::endl; - - while (fd < 0) - { - if (g_TCServer.Accept()) // try to get a connection - { - std::cerr << "Unrecoverable ERROR! Quiting..." << std::endl; - abort(); - } - // Either Accept timed out, or it's possile that other Transcoder ports were instead reopened - // So we'll check to see if this module is now open - fd = g_TCServer.GetFD(m_CSModule); - } - STCPacket pack; if (g_TCServer.Receive(m_CSModule, &pack, 8)) { diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index 1cfc3fc..433ab15 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -31,9 +31,9 @@ CReflector::CReflector() CReflector::~CReflector() { keep_running = false; - if ( m_XmlReportFuture.valid() ) + if ( m_MaintenanceFuture.valid() ) { - m_XmlReportFuture.get(); + m_MaintenanceFuture.get(); } for (auto it=m_Modules.cbegin(); it!=m_Modules.cend(); it++) @@ -131,7 +131,7 @@ bool CReflector::Start(void) // start the reporting thread try { - m_XmlReportFuture = std::async(std::launch::async, &CReflector::StateReportThread, this); + m_MaintenanceFuture = std::async(std::launch::async, &CReflector::MaintenanceThread, this); } catch(const std::exception& e) { @@ -155,9 +155,9 @@ void CReflector::Stop(void) g_TCServer.Close(); // stop & delete report threads - if ( m_XmlReportFuture.valid() ) + if ( m_MaintenanceFuture.valid() ) { - m_XmlReportFuture.get(); + m_MaintenanceFuture.get(); } // stop & delete all router thread @@ -333,12 +333,11 @@ void CReflector::RouterThread(const char ThisModule) } } -//////////////////////////////////////////////////////////////////////////////////////// -// report threads - +// Maintenance thread hands xml and/or json update, +// and also keeps the transcoder TCP port(s) connected #define XML_UPDATE_PERIOD 10 -void CReflector::StateReportThread() +void CReflector::MaintenanceThread() { std::string xmlpath, jsonpath; #ifndef NO_DHT @@ -348,6 +347,7 @@ void CReflector::StateReportThread() xmlpath.assign(g_Configure.GetString(g_Keys.files.xml)); if (g_Configure.Contains(g_Keys.files.json)) jsonpath.assign(g_Configure.GetString(g_Keys.files.json)); + auto tcport = g_Configure.GetUnsigned(g_Keys.tc.port); if (xmlpath.empty() && jsonpath.empty()) return; // nothing to do @@ -387,33 +387,41 @@ void CReflector::StateReportThread() } } +#ifndef NO_DHT + // update the dht data, if needed + if (peers_changed) + { + PutDHTPeers(); + peers_changed = false; + } + if (clients_changed) + { + PutDHTClients(); + clients_changed = false; + } + if (users_changed) + { + PutDHTUsers(); + users_changed = false; + } +#endif + // and wait a bit and do something useful at the same time - for (int i=0; i< XML_UPDATE_PERIOD && keep_running; i++) + for (int i=0; i< XML_UPDATE_PERIOD*10 && keep_running; i++) { -#ifndef NO_DHT - // update the dht data, if needed - if (peers_changed) - { - PutDHTPeers(); - peers_changed = false; - } - if (clients_changed) - { - PutDHTClients(); - clients_changed = false; - } - if (users_changed) + if (tcport && g_TCServer.AnyAreClosed()) { - PutDHTUsers(); - users_changed = false; + if (g_TCServer.Accept()) + { + std::cerr << "Unrecoverable error, aborting..." << std::endl; + abort(); + } } -#endif - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } } -//////////////////////////////////////////////////////////////////////////////////////// // notifications void CReflector::OnPeersChanged(void) diff --git a/reflector/Reflector.h b/reflector/Reflector.h index 5d8fb52..9b71b1e 100644 --- a/reflector/Reflector.h +++ b/reflector/Reflector.h @@ -94,7 +94,7 @@ protected: // threads void RouterThread(const char); - void StateReportThread(void); + void MaintenanceThread(void); // streams std::shared_ptr GetStream(char); @@ -121,7 +121,7 @@ protected: // threads std::atomic keep_running; std::unordered_map> m_RouterFuture; - std::future m_XmlReportFuture; + std::future m_MaintenanceFuture; #ifndef NO_DHT // Distributed Hash Table diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 24c7658..9d55e7a 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -63,13 +63,20 @@ void CTCSocket::Close(int fd) { if (fd == p.fd) { - if (close(p.fd)) + if (shutdown(p.fd, SHUT_RDWR)) { - std::cerr << "Error while closing " << fd << ": "; - perror("close"); + perror("shutdown"); } else - p.fd = -1; + { + if (close(p.fd)) + { + std::cerr << "Error while closing " << fd << ": "; + perror("close"); + } + else + p.fd = -1; + } return; } } @@ -96,7 +103,7 @@ char CTCSocket::GetMod(int fd) const return '?'; } -bool CTCServer::any_are_closed() +bool CTCServer::AnyAreClosed() const { for (auto &fds : m_Pfd) { @@ -171,7 +178,6 @@ bool CTCServer::Receive(char module, STCPacket *packet, int ms) auto pfds = &m_Pfd[pos]; if (pfds->fd < 0) { - std::cerr << "Can't receive on module '" << module << "' because it's closed" << std::endl; return rv; } @@ -265,10 +271,14 @@ bool CTCServer::Accept() return true; } - while (any_are_closed()) + while (AnyAreClosed()) { if (acceptone(fd)) + { + close(fd); + Close(); return true; + } } close(fd); diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 8a2de8a..381e3ae 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -58,11 +58,11 @@ public: bool Open(const std::string &address, const std::string &modules, uint16_t port); // Returns true if there is data bool Receive(char module, STCPacket *packet, int ms); + bool AnyAreClosed() const; bool Accept(); private: CIp m_Ip; - bool any_are_closed(); bool acceptone(int fd); }; From 76002325027250b250c68fc09c80bf63e9e974c7 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Wed, 22 May 2024 06:10:34 -0700 Subject: [PATCH 34/41] TC socket waiting messages --- reflector/TCSocket.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 9d55e7a..fcfa56b 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -271,6 +271,21 @@ bool CTCServer::Accept() return true; } + std::string wmod; + for (const char c : m_Modules) + { + if (GetFD(c) < 0) + wmod.append(1, c); + } + if (wmod.size() > 1) + { + std::cout << "Waiting for transcoder connections for modules " << wmod << "..." << std::endl; + } + else + { + std::cout << "Waiting for transcoder connection for module " << wmod << "..." << std::endl; + } + while (AnyAreClosed()) { if (acceptone(fd)) @@ -383,7 +398,7 @@ bool CTCClient::Connect(char module) { if (ECONNREFUSED == errno) { - if (0 == ++count % 100) std::cout << "Connection refused! Restart the system." << std::endl; + if (0 == ++count % 100) std::cout << "Connection refused! Restart the reflector." << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } else From 50fba705486556b17868854438dde2c852ad562b Mon Sep 17 00:00:00 2001 From: Tom Early Date: Wed, 22 May 2024 06:33:45 -0700 Subject: [PATCH 35/41] removed redundant msg --- reflector/TCSocket.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index fcfa56b..c8eaf96 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -231,8 +231,6 @@ bool CTCServer::Open(const std::string &address, const std::string &modules, uin pf.revents = 0; } - std::cout << "Waiting for " << m_Modules.size() << " transcoder connection(s) on " << m_Ip << "..." << std::endl; - return Accept(); } @@ -277,14 +275,17 @@ bool CTCServer::Accept() if (GetFD(c) < 0) wmod.append(1, c); } + + std::cout << "Waiting at " << m_Ip << " for transcoder connection"; if (wmod.size() > 1) { - std::cout << "Waiting for transcoder connections for modules " << wmod << "..." << std::endl; + std::cout << "s for modules " << wmod << "..." << std::endl; } else { - std::cout << "Waiting for transcoder connection for module " << wmod << "..." << std::endl; + std::cout << " for module " << wmod << "..." << std::endl; } + std::cout << wmod << "..." << std::endl; while (AnyAreClosed()) { From 4bbd3a860bc4d064477aa960ed54bb6b78c1dd44 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Wed, 22 May 2024 06:38:00 -0700 Subject: [PATCH 36/41] msg bug --- reflector/TCSocket.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index c8eaf96..44553bc 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -279,11 +279,11 @@ bool CTCServer::Accept() std::cout << "Waiting at " << m_Ip << " for transcoder connection"; if (wmod.size() > 1) { - std::cout << "s for modules " << wmod << "..." << std::endl; + std::cout << "s for modules "; } else { - std::cout << " for module " << wmod << "..." << std::endl; + std::cout << " for module "; } std::cout << wmod << "..." << std::endl; From 1d451bad560cb0562e24d8f27564a6fbf36c000d Mon Sep 17 00:00:00 2001 From: Tom Early Date: Wed, 22 May 2024 06:41:43 -0700 Subject: [PATCH 37/41] removed unnessary msg --- reflector/TCSocket.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 44553bc..5b8a357 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -155,7 +155,6 @@ bool CTCSocket::receive(int fd, STCPacket *packet) if (0 == n) { - std::cerr << "recv(): Module '" << GetMod(fd) << "' has been closed from the other side" << std::endl; return true; } From 23b76d3e95604233e73e5733f423b30babf20831 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Thu, 23 May 2024 06:39:13 -0700 Subject: [PATCH 38/41] readme --- README.md | 39 ++++++++++++++++++++------------------- reflector/TCSocket.cpp | 6 ++++-- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 91c2661..2f6b7e4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The URF Multi-protocol Gateway Reflector Server, ***urfd***, is part of the soft ## Introduction -This will build a new kind of digital voice reflector. A *urfd* supports DStar protocols (DPlus, DCS, DExtra and G3) DMR protocols (MMDVMHost, DMR+ and NXDN), M17, YSF, P25 (using IMBE) and USRP (Allstar). A key part of this is the hybrid transcoder, [tcd](https://github.com/n7tae/tcd), which is in a separate repository. You can't interlink urfd with xlxd. This reflector can be built without a transcoder, but clients will only hear other clients using the same codec. Please note that currently, urfd only supports the tcd transcoder when run locally. As a local device, urfd and tcd uses UNIX DGRAM sockets for inter-process communications. These kernel-base sockets are significantly faster than conventional UDP/IP sockets. It should be noted that tcd supports DVSI-3003 nad DVSI-3000 devices, which it uses for AMBE vocoding. +This will build a new kind of digital voice reflector. A *urfd* supports DStar protocols (DPlus, DCS, DExtra and G3) DMR protocols (MMDVMHost, DMR+ and NXDN), M17, YSF, P25 (using IMBE) and USRP (Allstar). A key part of this is the hybrid transcoder, [tcd](https://github.com/n7tae/tcd), which is in a separate repository. You can't interlink urfd with xlxd. This reflector can be built without a transcoder, but clients will only hear other clients using the same codec. This version of urfd/tcd now supports ***remote transcoding***. Communication between urfd and tcd is now handled *via* TCP, with *urfd* acting as the server and *tcd* is the client. In this way, it shouldn't be necessary to open any ports on the tcd system. A two-way TCP channel will be opened for each transcoded module. It should be noted that tcd supports DVSI-3003 nad DVSI-3000 devices, which it uses for AMBE vocoding. This build support *dual-stack* operation, so the server on which it's running, must have both an IPv4 and IPv6 routable address if you are going to configure a dual-stack reflector. @@ -33,7 +33,7 @@ The packages which are described in this document are designed to install server Below are instructions to build a URF reflector. If you are planning on an URF reflector without a transcoder, you can help your users by naming modules with names that suggest which protocol is welcome. You name modules in the config.inc.php file mentioned below. -The transcoder is in a separate repository, but you can install and monitor the transcoder and reflector from a script, *radmin* in this repository. You *should* look over the README.md file in the tcd repository to understand the transcoder. +For local transcoding reflectors only: The transcoder is in a separate repository, but you can install and monitor the transcoder and reflector from a script, *radmin* in this repository. You *should* look over the README.md file in the tcd repository to understand the transcoder. ### After a clean installation of Debian make sure to run update and upgrade @@ -48,6 +48,14 @@ sudo apt upgrade sudo apt install git apache2 php build-essential nlohmann-json3-dev libcurl4-gnutls-dev ``` +On the latest systems. *e.g.*, Ubuntu 24, Debian 12, you can install the OpenDHT support without having to build it. + +```bash +sudo apt install libopendht-dev +``` + +If the above command reports that this package is using something earlier that C++17, don't install it. You'll need to build it instead (described in the next section). + ### Ham-DHT support (optional, but highly recommended) **Ham-DHT**, a DHT network for hams, is implemented using a distributed hash table provided by OpenDHT. @@ -69,8 +77,6 @@ make sudo make install ``` -Please note that there is no easy way to uninstall OpenDHT once it's been installed. - ### Download and build the repository and ```bash @@ -112,7 +118,7 @@ After possibly editing `urfd.mk`, you can build your reflector: `make` . Besides Use your favorite text editor to set your run-time configuration in your copy of `urfd.ini`. -There are only a few things that need to be specified. Most important are, the reflector callsign and the IP addresses for the IPv4 and IPv6 listen ports and a transcoder port, if there is a transcoder. Dual-stack operation is enabled by specifying both an IPv4 and IPv6 address. IPv4-only single stack can be specified by leaving the IPv6 address undefined. +There are only a few things that need to be specified. Most important are, the reflector callsign and the IP addresses for the IPv4 and IPv6 listen ports and a transcoder port, if there is a transcoder. If you don't have a transcoder, be sure to set the transcoding port to zero. If you have a local transcoder, set the binding address to `127.0.0.1` or `::1`. If your transcoder is remote, you will usually set the binding address to `0.0.0.0` or `::`. Dual-stack operation is enabled by specifying both an IPv4 and IPv6 address. IPv4-only single stack can be specified by leaving the IPv6 address undefined. You can configure any modules, from **A** to **Z**. They don't have to be contiguous. If your reflector is configured with a transcoder, you can specify which configured modules will be transcoded. Up to three modules can be transcoded if you have the necessary hardware. @@ -150,7 +156,7 @@ The *dbutil* app can be used for several tasks relating to the three databases t - INIFILE is the path to the infile that defines the location of the http and file sources for these three databases. One at a time, *dbutil* can work with any of the three DATABASEs. It can read either the http or the file SOURCE. It can either show you the data entries that are syntactically correct or incorrect (ACTION). -### Installing your system +### Installing your system with a local transcoder After you have written your configuration files, you can install your system: @@ -160,14 +166,13 @@ After you have written your configuration files, you can install your system: You can use this interactive shell script to install and uninstall your system. This can also perform other tasks like restarting the reflector or transcoder process, or be used to view the reflector or transcoder log in real time. -### Stopping and starting the services manually +### Installing your system with a remote transcoder -```bash -sudo systemctl stop urfd # (or xrfd) -sudo systemctl stop tcd -``` - -You can start each component by replacing `stop` with `start`, or you can restart each by using `restart`. +In the `reflector` subdirectory, use: +- `make` to compile *urfd*. +- `sudo install` to install and start *urfd*. +- `systemctl` and `journalctl` can be used to manage *urfd* and view the log. +- `sudo make uninstall` will uninstall *urfd*. ### Copy dashboard to /var/www @@ -192,6 +197,7 @@ TCP port 80 (http) optional TCP port 443 (https) UDP port 8880 (DMR+ DMO mode) UDP port 10002 (BM connection) UDP port 10017 (URF interlinking) +TCP port 10100 (Transcoder listening port) UDP port 12345 - 12346 (G3 Icom Terminal presence and request port) UDP port 17000 (M17 protocol) UPD port 20001 (DPlus protocol) @@ -210,13 +216,8 @@ UDP port 62030 (MMDVM protocol) Pay attention, the URF Server acts as an YSF Master, which provides 26 wires-x rooms. It has nothing to do with the regular YSFReflector network, hence you don’t need to register your URF at ysfreflector.de! -## To-dos - -I will eventually support a remote transcoder option, so that you can, for example, run *urfd* in a data center, and then run the transcoder somewhere you have physical access so you can plug in your AMBE vocoders. I don't recommend this as it will add unnecessary and variable latency to your reflector. - -A new dashboard is on the to-do list! - ## Copyright - Copyright © 2016 Jean-Luc Deltombe LX3JL and Luc Engelmann LX1IQ - Copyright © 2022 Doug McLain AD8DP and Thomas A. Early N7TAE +- Copyright © 2024 Thomas A. Early N7TAE diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 5b8a357..524430b 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -388,7 +388,8 @@ bool CTCClient::Connect(char module) int yes = 1; if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int))) { - perror("TC client setsockopt"); + std::cerr << "Moudule " << module << " error:"; + perror("setsockopt"); close(fd); return true; } @@ -403,7 +404,7 @@ bool CTCClient::Connect(char module) } else { - std::cerr << "For module '" << module << "' : "; + std::cerr << "Module " << module << " error: "; perror("connect"); close(fd); return true; @@ -439,6 +440,7 @@ bool CTCClient::ReConnect() { if (-1 == GetFD(m)) { + std::cout << "Reconnecting module " << m << "..." << std::endl; if (Connect(m)) { rv = true; From 3ee6f519f96b4520a266668c5d04cb3876dd9bf8 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Fri, 24 May 2024 06:09:37 -0700 Subject: [PATCH 39/41] if Accept fails, raise SIGINT to stop --- reflector/TCSocket.cpp | 9 ++++----- reflector/TCSocket.h | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/reflector/TCSocket.cpp b/reflector/TCSocket.cpp index 524430b..7f27a9d 100644 --- a/reflector/TCSocket.cpp +++ b/reflector/TCSocket.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -433,21 +434,19 @@ bool CTCClient::Connect(char module) return false; } -bool CTCClient::ReConnect() +void CTCClient::ReConnect() { - bool rv = false; for (char m : m_Modules) { - if (-1 == GetFD(m)) + if (0 > GetFD(m)) { std::cout << "Reconnecting module " << m << "..." << std::endl; if (Connect(m)) { - rv = true; + raise(SIGINT); } } } - return rv; } void CTCClient::Receive(std::queue> &queue, int ms) diff --git a/reflector/TCSocket.h b/reflector/TCSocket.h index 381e3ae..f6ffaac 100644 --- a/reflector/TCSocket.h +++ b/reflector/TCSocket.h @@ -73,7 +73,7 @@ public: ~CTCClient() {} bool Open(const std::string &address, const std::string &modules, uint16_t port); void Receive(std::queue> &queue, int ms); - bool ReConnect(); + void ReConnect(); private: std::string m_Address; From c99492440ebf15b2d527282fb2ce6de6326592a0 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Mon, 3 Jun 2024 06:00:06 -0700 Subject: [PATCH 40/41] allow tc port to be zero --- reflector/Configure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reflector/Configure.cpp b/reflector/Configure.cpp index fa29100..75e28d8 100644 --- a/reflector/Configure.cpp +++ b/reflector/Configure.cpp @@ -290,7 +290,7 @@ bool CConfigure::ReadData(const std::string &path) break; case ESection::tc: if (0 == key.compare(JPORT)) - data[g_Keys.tc.port] = getUnsigned(value, "Transcoder Port", 1024, 40000, 10100); + data[g_Keys.tc.port] = getUnsigned(value, "Transcoder Port", 0, 40000, 10100); else if (0 == key.compare(JBINDINGADDRESS)) data[g_Keys.tc.bind] = value; else if (0 == key.compare(JMODULES)) From 5519b5ca4a0b2e7a39e9dbbec8592e4e9df17ec5 Mon Sep 17 00:00:00 2001 From: Tom Early Date: Sat, 8 Jun 2024 09:47:51 -0700 Subject: [PATCH 41/41] dht is config data only --- reflector/Clients.cpp | 4 - reflector/Peers.cpp | 5 -- reflector/Reflector.cpp | 139 +----------------------------- reflector/Reflector.h | 8 -- reflector/Users.cpp | 3 - reflector/dht-values.h | 183 ++++++++++++++++++---------------------- 6 files changed, 85 insertions(+), 257 deletions(-) diff --git a/reflector/Clients.cpp b/reflector/Clients.cpp index 4e0bca4..74e7b66 100644 --- a/reflector/Clients.cpp +++ b/reflector/Clients.cpp @@ -63,8 +63,6 @@ void CClients::AddClient(std::shared_ptr client) std::cout << " on module " << client->GetReflectorModule(); } std::cout << std::endl; - // notify - g_Reflector.OnClientsChanged(); } void CClients::RemoveClient(std::shared_ptr client) @@ -87,8 +85,6 @@ void CClients::RemoveClient(std::shared_ptr client) } std::cout << std::endl; m_Clients.erase(it); - // notify - g_Reflector.OnClientsChanged(); break; } } diff --git a/reflector/Peers.cpp b/reflector/Peers.cpp index 9be9c04..4404158 100644 --- a/reflector/Peers.cpp +++ b/reflector/Peers.cpp @@ -64,9 +64,6 @@ void CPeers::AddPeer(std::shared_ptr peer) clients->AddClient(*cit); } g_Reflector.ReleaseClients(); - - // notify - g_Reflector.OnPeersChanged(); } void CPeers::RemovePeer(std::shared_ptr peer) @@ -92,8 +89,6 @@ void CPeers::RemovePeer(std::shared_ptr peer) // remove it std::cout << "Peer " << (*pit)->GetCallsign() << " at " << (*pit)->GetIp() << " removed" << std::endl; pit = m_Peers.erase(pit); - // notify - g_Reflector.OnPeersChanged(); } else { diff --git a/reflector/Reflector.cpp b/reflector/Reflector.cpp index 433ab15..4861ae8 100644 --- a/reflector/Reflector.cpp +++ b/reflector/Reflector.cpp @@ -21,12 +21,7 @@ #include "Global.h" -CReflector::CReflector() -{ -#ifndef NO_DHT - peers_put_count = clients_put_count = users_put_count = 0; -#endif -} +CReflector::CReflector() {} CReflector::~CReflector() { @@ -340,9 +335,6 @@ void CReflector::RouterThread(const char ThisModule) void CReflector::MaintenanceThread() { std::string xmlpath, jsonpath; -#ifndef NO_DHT - peers_changed = clients_changed = users_changed = true; -#endif if (g_Configure.Contains(g_Keys.files.xml)) xmlpath.assign(g_Configure.GetString(g_Keys.files.xml)); if (g_Configure.Contains(g_Keys.files.json)) @@ -387,24 +379,6 @@ void CReflector::MaintenanceThread() } } -#ifndef NO_DHT - // update the dht data, if needed - if (peers_changed) - { - PutDHTPeers(); - peers_changed = false; - } - if (clients_changed) - { - PutDHTClients(); - clients_changed = false; - } - if (users_changed) - { - PutDHTUsers(); - users_changed = false; - } -#endif // and wait a bit and do something useful at the same time for (int i=0; i< XML_UPDATE_PERIOD*10 && keep_running; i++) @@ -422,29 +396,6 @@ void CReflector::MaintenanceThread() } } -// notifications - -void CReflector::OnPeersChanged(void) -{ -#ifndef NO_DHT - peers_changed = true; -#endif -} - -void CReflector::OnClientsChanged(void) -{ -#ifndef NO_DHT - clients_changed = true; -#endif -} - -void CReflector::OnUsersChanged(void) -{ -#ifndef NO_DHT - users_changed = true; -#endif -} - //////////////////////////////////////////////////////////////////////////////////////// // modules & queues @@ -563,94 +514,6 @@ void CReflector::WriteXmlFile(std::ofstream &xmlFile) #ifndef NO_DHT // DHT put() and get() -void CReflector::PutDHTPeers() -{ - const std::string cs(g_Configure.GetString(g_Keys.names.callsign)); - // load it up - SUrfdPeers1 p; - time(&p.timestamp); - p.sequence = peers_put_count++; - auto peers = GetPeers(); - for (auto pit=peers->cbegin(); pit!=peers->cend(); pit++) - { - p.list.emplace_back((*pit)->GetCallsign().GetCS(), (*pit)->GetReflectorModules(), (*pit)->GetConnectTime()); - } - ReleasePeers(); - - auto nv = std::make_shared(p); - nv->user_type.assign(URFD_PEERS_1); - nv->id = toUType(EUrfdValueID::Peers); - - node.putSigned( - refhash, - nv, -#ifdef DEBUG - [](bool success){ std::cout << "PutDHTPeers() " << (success ? "successful" : "unsuccessful") << std::endl; }, -#else - [](bool success){ if (! success) std::cout << "PutDHTPeers() unsuccessful" << std::endl; }, -#endif - true // permanent! - ); -} - -void CReflector::PutDHTClients() -{ - const std::string cs(g_Configure.GetString(g_Keys.names.callsign)); - SUrfdClients1 c; - time(&c.timestamp); - c.sequence = clients_put_count++; - auto clients = GetClients(); - for (auto cit=clients->cbegin(); cit!=clients->cend(); cit++) - { - c.list.emplace_back((*cit)->GetCallsign().GetCS(), std::string((*cit)->GetIp().GetAddress()), (*cit)->GetReflectorModule(), (*cit)->GetConnectTime(), (*cit)->GetLastHeardTime()); - } - ReleaseClients(); - - auto nv = std::make_shared(c); - nv->user_type.assign(URFD_CLIENTS_1); - nv->id = toUType(EUrfdValueID::Clients); - - node.putSigned( - refhash, - nv, -#ifdef DEBUG - [](bool success){ std::cout << "PutDHTClients() " << (success ? "successful" : "unsuccessful") << std::endl; }, -#else - [](bool success){ if (! success) std::cout << "PutDHTClients() unsuccessful" << std::endl; }, -#endif - false // not permanent! - ); -} - -void CReflector::PutDHTUsers() -{ - const std::string cs(g_Configure.GetString(g_Keys.names.callsign)); - SUrfdUsers1 u; - time(&u.timestamp); - u.sequence = users_put_count++; - auto users = GetUsers(); - for (auto uit=users->cbegin(); uit!=users->cend(); uit++) - { - u.list.emplace_back((*uit).GetCallsign(), std::string((*uit).GetViaNode()), (*uit).GetOnModule(), (*uit).GetViaPeer(), (*uit).GetLastHeardTime()); - } - ReleaseUsers(); - - auto nv = std::make_shared(u); - nv->user_type.assign(URFD_USERS_1); - nv->id = toUType(EUrfdValueID::Users); - - node.putSigned( - refhash, - nv, -#ifdef DEBUG - [](bool success){ std::cout << "PutDHTUsers() " << (success ? "successful" : "unsuccessful") << std::endl; }, -#else - [](bool success){ if (! success) std::cout << "PutDHTUsers() unsuccessful" << std::endl; }, -#endif - false // not permanent - ); -} - void CReflector::PutDHTConfig() { const std::string cs(g_Configure.GetString(g_Keys.names.callsign)); diff --git a/reflector/Reflector.h b/reflector/Reflector.h index 9b71b1e..dd260f3 100644 --- a/reflector/Reflector.h +++ b/reflector/Reflector.h @@ -76,9 +76,6 @@ public: // notifications - void OnPeersChanged(void); - void OnClientsChanged(void); - void OnUsersChanged(void); #ifndef NO_DHT void GetDHTConfig(const std::string &cs); #endif @@ -87,9 +84,6 @@ protected: #ifndef NO_DHT // Publish DHT void PutDHTConfig(); - void PutDHTPeers(); - void PutDHTClients(); - void PutDHTUsers(); #endif // threads @@ -127,7 +121,5 @@ protected: // Distributed Hash Table dht::DhtRunner node; dht::InfoHash refhash; - unsigned int peers_put_count, clients_put_count, users_put_count; - std::atomic peers_changed, clients_changed, users_changed; #endif }; diff --git a/reflector/Users.cpp b/reflector/Users.cpp index ac7d774..31f4591 100644 --- a/reflector/Users.cpp +++ b/reflector/Users.cpp @@ -39,9 +39,6 @@ void CUsers::AddUser(const CUser &user) { m_Users.resize(m_Users.size()-1); } - - // notify - g_Reflector.OnUsersChanged(); } //////////////////////////////////////////////////////////////////////////////////////// diff --git a/reflector/dht-values.h b/reflector/dht-values.h index f0fed6f..b6ff056 100644 --- a/reflector/dht-values.h +++ b/reflector/dht-values.h @@ -1,28 +1,32 @@ -// Copyright © 2022 Thomas A. Early, N7TAE -// -// ---------------------------------------------------------------------------- -// This is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// with this software. If not, see . -// ---------------------------------------------------------------------------- +/* + * Copyright (c) 2022-2024 by Thomas A. Early N7TAE + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ #pragma once #include -#define USE_MREFD_VALUES +// comment out sections you don't need for your application +//#define USE_MREFD_VALUES #define USE_URFD_VALUES -/* HELPERS */ +// a typesafe way to extract the numeric value from a enum class +// note that this is a constexpr and so can even be used in an +// array declaration or as a tuple index template constexpr auto toUType(E enumerator) noexcept { return static_cast>(enumerator); @@ -30,112 +34,93 @@ template constexpr auto toUType(E enumerator) noexcept #ifdef USE_MREFD_VALUES -// user_type for mrefd values -#define MREFD_USERS_1 "mrefd-users-1" -#define MREFD_PEERS_1 "mrefd-peers-1" +// DHT::value::user_type for mrefd values +// These are the value release version strings in a +// preprocessor definition for your convience +// if your app needs a particular part, then it should handle all versions of that part #define MREFD_CONFIG_1 "mrefd-config-1" -#define MREFD_CLIENTS_1 "mrefd-clients-1" -// dht::Value ids of the different parts of the document +// dht::Value ids of the different parts of the mrefd document // can be assigned any unsigned value except 0 +// more parts can be added, but don't change the value of any existing part +// using toUType, you can set or query a user_type to determine the value part +// this can be done before unpacking the MSGPACK enum class EMrefdValueID : uint64_t { Config=1, Peers=2, Clients=3, Users=4 }; -using MrefdPeerTuple = std::tuple; -enum class EMrefdPeerFields { Callsign, Modules, ConnectTime }; -struct SMrefdPeers1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; - -using MrefdClientTuple = std::tuple; -enum class EMrefdClientFields { Callsign, Ip, Module, ConnectTime, LastHeardTime }; -struct SMrefdClients1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; - -using MrefdUserTuple = std::tuple; -enum class EMrefdUserFields { Source, Destination, Reflector, LastHeardTime }; -struct SMrefdUsers1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; +///////////////// MREFD PART VALUES /////////////// -struct SMrefdConfig1 +// the configuration part +struct SMrefdConfig1 // user_type is MREFD_CONFIG_1 { - std::time_t timestamp; - std::string callsign, ipv4addr, ipv6addr, modules, encryptedmods, url, email, sponsor, country, version; - uint16_t port; - + std::time_t timestamp; // when this value was set + std::string callsign; // the callsign of the mrefd reflector + std::string ipv4addr; // the external IPv4 address + std::string ipv6addr; // the external IPv6 address + std::string modules; // all the configured modules, [A-Z] + std::string encryptedmods; // modules that will pass encrypted streams + std::string url; // the URL of the dashboard + std::string email; // the email of the responsible owner + std::string sponsor; // the organization or individual sponsoring the reflector + std::string country; // the 2-letter country code + std::string version; // the version of the reflector software + uint16_t port; // the connection listening UDP port, usually 17000 + + // the order in which MSGPACK serializes the data MSGPACK_DEFINE(timestamp, callsign, ipv4addr, ipv6addr, modules, encryptedmods, url, email, sponsor, country, version, port) }; -#endif +#endif // USE_MREFD_VALUES #ifdef USE_URFD_VALUES -#define URFD_PEERS_1 "urfd-peers-1" -#define URFD_USERS_1 "urfd-users-1" +// DHT::value::user_type for urfd values +// These are the value release version strings in a +// preprocessor definition for your convience +// if your app needs a particular part, then it should handle all versions of that part #define URFD_CONFIG_1 "urfd-config-1" -#define URFD_CLIENTS_1 "urfd-clients-1" +#define URFD_CONFIG_2 "urfd-config-2" +// dht::Value::id of the different parts of the urfd document +// can be assigned any unsigned value except 0 +// more parts can be added, but don't change the value of any existing part +// using toUType, you can set or query a user_type to determine the value part +// this can be done before unpacking the MSGPACK enum class EUrfdValueID : uint64_t { Config=1, Peers=2, Clients=3, Users=4 }; -using UrfdPeerTuple = std::tuple; -enum class EUrfdPeerFields { Callsign, Modules, ConnectTime }; -struct SUrfdPeers1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; - -using UrfdClientTuple = std::tuple; -enum class EUrfdClientFields { Callsign, Ip, Module, ConnectTime, LastHeardTime }; -struct SUrfdClients1 +// the following enum classes can be used to reference a particular value in a fixed array +// 'SIZE' has to be last value for these scoped enums as this is used to declare these arrays +// +// all the configurable ports in urfd (G3 and BM are not configurable) +enum class EUrfdPorts : unsigned { dcs, dextra, dmrplus, dplus, m17, mmdvm, nxdn, p25, urf, ysf, SIZE }; +// autolink modules for these protocols +enum class EUrfdAlMod : unsigned { nxdn, p25, ysf, SIZE }; +// default TX/RX values for ysf +enum class EUrfdTxRx : unsigned { rx, tx, SIZE }; +// reflector ID values for these two modes +enum class EUrfdRefId : unsigned { nxdn, p25, SIZE }; + +struct SUrfdConfig1 // user_type is URFD_CONFIG_1 { std::time_t timestamp; - unsigned int sequence; - std::list list; + std::string callsign, ipv4addr, ipv6addr, modules, transcodedmods, url, email, sponsor, country, version; + // transcodedmods are those modules that support full transcoding + std::array port; + std::array almod; + std::array ysffreq; + std::array refid; + std::unordered_map description; + bool g3enabled; - MSGPACK_DEFINE(timestamp, sequence, list) + MSGPACK_DEFINE(timestamp, callsign, ipv4addr, ipv6addr, modules, transcodedmods, url, email, sponsor, country, version, almod, ysffreq, refid, g3enabled, port, description) }; -using UrfdUserTuple = std::tuple; -enum class EUrfdUserFields { Callsign, ViaNode, OnModule, ViaPeer, LastHeardTime }; -struct SUrfdUsers1 -{ - std::time_t timestamp; - unsigned int sequence; - std::list list; - - MSGPACK_DEFINE(timestamp, sequence, list) -}; +enum class EUrfdPorts2 : unsigned { dcs, dextra, dmrplus, dplus, dsd, m17, mmdvm, nxdn, p25, urf, ysf, SIZE }; -// 'SIZE' has to be last value for these scoped enums -enum class EUrfdPorts : unsigned { dcs, dextra, dmrplus, dplus, m17, mmdvm, nxdn, p25, urf, ysf, SIZE }; -enum class EUrfdAlMod : unsigned { nxdn, p25, ysf, SIZE }; -enum class EUrfdTxRx : unsigned { rx, tx, SIZE }; -enum class EUrfdRefId : unsigned { nxdn, p25, SIZE }; -struct SUrfdConfig1 +struct SUrfdConfig2 { std::time_t timestamp; std::string callsign, ipv4addr, ipv6addr, modules, transcodedmods, url, email, sponsor, country, version; - std::array port; + std::array port; std::array almod; std::array ysffreq; std::array refid; @@ -145,4 +130,4 @@ struct SUrfdConfig1 MSGPACK_DEFINE(timestamp, callsign, ipv4addr, ipv6addr, modules, transcodedmods, url, email, sponsor, country, version, almod, ysffreq, refid, g3enabled, port, description) }; -#endif +#endif // USE_URFD_VALUES