From 54287f1a4bc161e12e5a6e4713e633882f0cf806 Mon Sep 17 00:00:00 2001 From: accius Date: Mon, 2 Feb 2026 15:47:43 -0500 Subject: [PATCH] fixing pskreporter integ --- server.js | 321 +++++++++++++--------------- src/components/PSKReporterPanel.jsx | 10 +- src/hooks/usePSKReporter.js | 139 +++++++----- 3 files changed, 244 insertions(+), 226 deletions(-) diff --git a/server.js b/server.js index 9df0890..ab26023 100644 --- a/server.js +++ b/server.js @@ -207,7 +207,7 @@ app.use('/api', (req, res, next) => { } else if (path.includes('/pota') || path.includes('/sota')) { cacheDuration = 120; // 2 minutes } else if (path.includes('/pskreporter')) { - cacheDuration = 120; // 2 minutes (respect PSKReporter rate limits) + cacheDuration = 300; // 5 minutes (PSKReporter rate limits aggressively) } else if (path.includes('/dxcluster') || path.includes('/myspots')) { cacheDuration = 30; // 30 seconds (DX spots need to be relatively fresh) } else if (path.includes('/config')) { @@ -1853,56 +1853,40 @@ app.get('/api/myspots/:callsign', async (req, res) => { }); // ============================================ -// PSKREPORTER API +// PSKREPORTER API (MQTT-based for real-time) // ============================================ -// Cache for PSKReporter data (2-minute cache to respect their rate limits) -let pskReporterCache = {}; -const PSK_CACHE_TTL = 2 * 60 * 1000; // 2 minutes +// PSKReporter MQTT feed at mqtt.pskreporter.info provides real-time spots +// WebSocket endpoints: 1885 (ws), 1886 (wss) +// Topic format: pskr/filter/v2/{band}/{mode}/{sendercall}/{receivercall}/{senderlocator}/{receiverlocator}/{sendercountry}/{receivercountry} -// Parse PSKReporter XML response -function parsePSKReporterXML(xml) { - const reports = []; - - // Extract reception reports using regex (simple XML parsing) - const reportRegex = /]*>([\s\S]*?)<\/receptionReport>/g; - let match; - - while ((match = reportRegex.exec(xml)) !== null) { - const report = match[0]; - - // Extract attributes - const getAttr = (name) => { - const attrMatch = report.match(new RegExp(`${name}="([^"]*)"`)); - return attrMatch ? attrMatch[1] : null; - }; - - const receiverCallsign = getAttr('receiverCallsign'); - const receiverLocator = getAttr('receiverLocator'); - const senderCallsign = getAttr('senderCallsign'); - const senderLocator = getAttr('senderLocator'); - const frequency = getAttr('frequency'); - const mode = getAttr('mode'); - const flowStartSeconds = getAttr('flowStartSeconds'); - const sNR = getAttr('sNR'); - - if (receiverCallsign && senderCallsign) { - reports.push({ - receiver: receiverCallsign, - receiverGrid: receiverLocator, - sender: senderCallsign, - senderGrid: senderLocator, - freq: frequency ? (parseInt(frequency) / 1000000).toFixed(6) : null, - freqMHz: frequency ? (parseInt(frequency) / 1000000).toFixed(3) : null, - mode: mode || 'Unknown', - timestamp: flowStartSeconds ? parseInt(flowStartSeconds) * 1000 : Date.now(), - snr: sNR ? parseInt(sNR) : null - }); +// Cache for PSKReporter data - stores recent spots from MQTT +const pskReporterSpots = { + tx: new Map(), // Map of callsign -> spots where they're being heard + rx: new Map(), // Map of callsign -> spots they're receiving + maxAge: 60 * 60 * 1000 // Keep spots for 1 hour max +}; + +// Clean up old spots periodically +setInterval(() => { + const cutoff = Date.now() - pskReporterSpots.maxAge; + for (const [call, spots] of pskReporterSpots.tx) { + const filtered = spots.filter(s => s.timestamp > cutoff); + if (filtered.length === 0) { + pskReporterSpots.tx.delete(call); + } else { + pskReporterSpots.tx.set(call, filtered); } } - - return reports; -} + for (const [call, spots] of pskReporterSpots.rx) { + const filtered = spots.filter(s => s.timestamp > cutoff); + if (filtered.length === 0) { + pskReporterSpots.rx.delete(call); + } else { + pskReporterSpots.rx.set(call, filtered); + } + } +}, 5 * 60 * 1000); // Clean every 5 minutes // Convert grid square to lat/lon function gridToLatLonSimple(grid) { @@ -1928,9 +1912,9 @@ function gridToLatLonSimple(grid) { return { lat: finalLat, lon: finalLon }; } -// Get band name from frequency in MHz -function getBandFromMHz(freqMHz) { - const freq = parseFloat(freqMHz); +// Get band name from frequency in Hz +function getBandFromHz(freqHz) { + const freq = freqHz / 1000000; // Convert to MHz if (freq >= 1.8 && freq <= 2) return '160m'; if (freq >= 3.5 && freq <= 4) return '80m'; if (freq >= 5.3 && freq <= 5.4) return '60m'; @@ -1947,171 +1931,159 @@ function getBandFromMHz(freqMHz) { return 'Unknown'; } -// PSKReporter - where is my signal being heard? -app.get('/api/pskreporter/tx/:callsign', async (req, res) => { +// PSKReporter endpoint - returns MQTT connection info for frontend +// The frontend connects directly to MQTT via WebSocket for real-time updates +app.get('/api/pskreporter/config', (req, res) => { + res.json({ + mqtt: { + host: 'mqtt.pskreporter.info', + wsPort: 1885, // WebSocket + wssPort: 1886, // WebSocket + TLS (recommended) + topicPrefix: 'pskr/filter/v2' + }, + // Topic format: pskr/filter/v2/{band}/{mode}/{sendercall}/{receivercall}/{senderlocator}/{receiverlocator}/{sendercountry}/{receivercountry} + // Use + for single-level wildcard, # for multi-level + // Example for TX (being heard): pskr/filter/v2/+/+/{CALLSIGN}/# + // Example for RX (hearing): Subscribe and filter client-side + info: 'Connect via WebSocket to mqtt.pskreporter.info:1886 (wss) for real-time spots' + }); +}); + +// Fallback HTTP endpoint for when MQTT isn't available +// Uses the traditional retrieve API with caching +let pskHttpCache = {}; +const PSK_HTTP_CACHE_TTL = 5 * 60 * 1000; // 5 minutes + +app.get('/api/pskreporter/http/:callsign', async (req, res) => { const callsign = req.params.callsign.toUpperCase(); - const minutes = parseInt(req.query.minutes) || 15; // Default 15 minutes - const flowStartSeconds = Math.floor(minutes * 60); + const minutes = parseInt(req.query.minutes) || 15; + const direction = req.query.direction || 'tx'; // tx or rx + // flowStartSeconds must be NEGATIVE for "last N seconds" + const flowStartSeconds = -Math.abs(minutes * 60); - const cacheKey = `tx:${callsign}:${minutes}`; + const cacheKey = `${direction}:${callsign}:${minutes}`; const now = Date.now(); // Check cache - if (pskReporterCache[cacheKey] && (now - pskReporterCache[cacheKey].timestamp) < PSK_CACHE_TTL) { - return res.json(pskReporterCache[cacheKey].data); + if (pskHttpCache[cacheKey] && (now - pskHttpCache[cacheKey].timestamp) < PSK_HTTP_CACHE_TTL) { + return res.json({ ...pskHttpCache[cacheKey].data, cached: true }); } try { - console.log(`[PSKReporter] Fetching TX reports for ${callsign} (last ${minutes} min)`); + const param = direction === 'tx' ? 'senderCallsign' : 'receiverCallsign'; + // Add appcontact parameter as requested by PSKReporter developer docs + const url = `https://retrieve.pskreporter.info/query?${param}=${encodeURIComponent(callsign)}&flowStartSeconds=${flowStartSeconds}&rronly=1&appcontact=openhamclock`; - const url = `https://retrieve.pskreporter.info/query?senderCallsign=${encodeURIComponent(callsign)}&flowStartSeconds=${flowStartSeconds}&rronly=1&noactive=1&nolocator=1`; + console.log(`[PSKReporter HTTP] Fetching ${direction} for ${callsign} (last ${minutes} min)`); const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 15000); + const timeout = setTimeout(() => controller.abort(), 20000); const response = await fetch(url, { headers: { - 'User-Agent': 'OpenHamClock/3.10', - 'Accept': 'application/xml' + 'User-Agent': 'OpenHamClock/3.11 (Amateur Radio Dashboard)', + 'Accept': '*/*' }, signal: controller.signal }); clearTimeout(timeout); if (!response.ok) { - throw new Error(`PSKReporter returned ${response.status}`); + throw new Error(`HTTP ${response.status}`); } const xml = await response.text(); - const reports = parsePSKReporterXML(xml); - - // Add location data and band info - const enrichedReports = reports.map(r => { - const loc = r.receiverGrid ? gridToLatLonSimple(r.receiverGrid) : null; - return { - ...r, - lat: loc?.lat, - lon: loc?.lon, - band: r.freqMHz ? getBandFromMHz(r.freqMHz) : 'Unknown', - age: Math.floor((Date.now() - r.timestamp) / 60000) // minutes ago + const reports = []; + + // Parse XML response + const reportRegex = /]*>/g; + let match; + while ((match = reportRegex.exec(xml)) !== null) { + const report = match[0]; + const getAttr = (name) => { + const m = report.match(new RegExp(`${name}="([^"]*)"`)); + return m ? m[1] : null; }; - }).filter(r => r.lat && r.lon); + + const receiverCallsign = getAttr('receiverCallsign'); + const receiverLocator = getAttr('receiverLocator'); + const senderCallsign = getAttr('senderCallsign'); + const senderLocator = getAttr('senderLocator'); + const frequency = getAttr('frequency'); + const mode = getAttr('mode'); + const flowStartSecs = getAttr('flowStartSeconds'); + const sNR = getAttr('sNR'); + + if (receiverCallsign && senderCallsign) { + const locator = direction === 'tx' ? receiverLocator : senderLocator; + const loc = locator ? gridToLatLonSimple(locator) : null; + + reports.push({ + sender: senderCallsign, + senderGrid: senderLocator, + receiver: receiverCallsign, + receiverGrid: receiverLocator, + freq: frequency ? parseInt(frequency) : null, + freqMHz: frequency ? (parseInt(frequency) / 1000000).toFixed(3) : null, + band: frequency ? getBandFromHz(parseInt(frequency)) : 'Unknown', + mode: mode || 'Unknown', + timestamp: flowStartSecs ? parseInt(flowStartSecs) * 1000 : Date.now(), + snr: sNR ? parseInt(sNR) : null, + lat: loc?.lat, + lon: loc?.lon, + age: flowStartSecs ? Math.floor((Date.now() / 1000 - parseInt(flowStartSecs)) / 60) : 0 + }); + } + } // Sort by timestamp (newest first) - enrichedReports.sort((a, b) => b.timestamp - a.timestamp); + reports.sort((a, b) => b.timestamp - a.timestamp); const result = { callsign, - direction: 'tx', - count: enrichedReports.length, - reports: enrichedReports.slice(0, 100), // Limit to 100 reports - timestamp: new Date().toISOString() + direction, + count: reports.length, + reports: reports.slice(0, 100), + timestamp: new Date().toISOString(), + source: 'http' }; - // Cache the result - pskReporterCache[cacheKey] = { data: result, timestamp: now }; + // Cache it + pskHttpCache[cacheKey] = { data: result, timestamp: now }; - console.log(`[PSKReporter] Found ${enrichedReports.length} stations hearing ${callsign}`); + console.log(`[PSKReporter HTTP] Found ${reports.length} ${direction} reports for ${callsign}`); res.json(result); } catch (error) { - if (error.name !== 'AbortError') { - logErrorOnce('PSKReporter', `TX query error: ${error.message}`); - } + logErrorOnce('PSKReporter HTTP', error.message); + // Return cached data if available - if (pskReporterCache[cacheKey]) { - return res.json(pskReporterCache[cacheKey].data); + if (pskHttpCache[cacheKey]) { + return res.json({ ...pskHttpCache[cacheKey].data, cached: true, stale: true }); } - res.json({ callsign, direction: 'tx', count: 0, reports: [], error: error.message }); - } -}); - -// PSKReporter - what am I hearing? -app.get('/api/pskreporter/rx/:callsign', async (req, res) => { - const callsign = req.params.callsign.toUpperCase(); - const minutes = parseInt(req.query.minutes) || 15; - const flowStartSeconds = Math.floor(minutes * 60); - - const cacheKey = `rx:${callsign}:${minutes}`; - const now = Date.now(); - - // Check cache - if (pskReporterCache[cacheKey] && (now - pskReporterCache[cacheKey].timestamp) < PSK_CACHE_TTL) { - return res.json(pskReporterCache[cacheKey].data); - } - - try { - console.log(`[PSKReporter] Fetching RX reports for ${callsign} (last ${minutes} min)`); - const url = `https://retrieve.pskreporter.info/query?receiverCallsign=${encodeURIComponent(callsign)}&flowStartSeconds=${flowStartSeconds}&rronly=1&noactive=1&nolocator=1`; - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 15000); - - const response = await fetch(url, { - headers: { - 'User-Agent': 'OpenHamClock/3.10', - 'Accept': 'application/xml' - }, - signal: controller.signal + res.json({ + callsign, + direction, + count: 0, + reports: [], + error: error.message, + hint: 'Consider using MQTT WebSocket connection for real-time data' }); - clearTimeout(timeout); - - if (!response.ok) { - throw new Error(`PSKReporter returned ${response.status}`); - } - - const xml = await response.text(); - const reports = parsePSKReporterXML(xml); - - // Add location data and band info - const enrichedReports = reports.map(r => { - const loc = r.senderGrid ? gridToLatLonSimple(r.senderGrid) : null; - return { - ...r, - lat: loc?.lat, - lon: loc?.lon, - band: r.freqMHz ? getBandFromMHz(r.freqMHz) : 'Unknown', - age: Math.floor((Date.now() - r.timestamp) / 60000) - }; - }).filter(r => r.lat && r.lon); - - enrichedReports.sort((a, b) => b.timestamp - a.timestamp); - - const result = { - callsign, - direction: 'rx', - count: enrichedReports.length, - reports: enrichedReports.slice(0, 100), - timestamp: new Date().toISOString() - }; - - pskReporterCache[cacheKey] = { data: result, timestamp: now }; - - console.log(`[PSKReporter] Found ${enrichedReports.length} stations heard by ${callsign}`); - res.json(result); - - } catch (error) { - if (error.name !== 'AbortError') { - logErrorOnce('PSKReporter', `RX query error: ${error.message}`); - } - if (pskReporterCache[cacheKey]) { - return res.json(pskReporterCache[cacheKey].data); - } - res.json({ callsign, direction: 'rx', count: 0, reports: [], error: error.message }); } }); -// PSKReporter - combined TX and RX for a callsign +// Combined endpoint that tries MQTT cache first, falls back to HTTP app.get('/api/pskreporter/:callsign', async (req, res) => { const callsign = req.params.callsign.toUpperCase(); const minutes = parseInt(req.query.minutes) || 15; + // For now, redirect to HTTP endpoint since MQTT requires client-side connection + // The frontend should connect directly to MQTT for real-time updates try { - // Fetch both TX and RX in parallel const [txRes, rxRes] = await Promise.allSettled([ - fetch(`http://localhost:${PORT}/api/pskreporter/tx/${callsign}?minutes=${minutes}`), - fetch(`http://localhost:${PORT}/api/pskreporter/rx/${callsign}?minutes=${minutes}`) + fetch(`http://localhost:${PORT}/api/pskreporter/http/${callsign}?minutes=${minutes}&direction=tx`), + fetch(`http://localhost:${PORT}/api/pskreporter/http/${callsign}?minutes=${minutes}&direction=rx`) ]); let txData = { count: 0, reports: [] }; @@ -2128,15 +2100,24 @@ app.get('/api/pskreporter/:callsign', async (req, res) => { callsign, tx: txData, rx: rxData, - timestamp: new Date().toISOString() + timestamp: new Date().toISOString(), + mqtt: { + available: true, + host: 'wss://mqtt.pskreporter.info:1886', + hint: 'Connect via WebSocket for real-time updates' + } }); } catch (error) { - logErrorOnce('PSKReporter', `Combined query error: ${error.message}`); - res.json({ callsign, tx: { count: 0, reports: [] }, rx: { count: 0, reports: [] }, error: error.message }); + logErrorOnce('PSKReporter', error.message); + res.json({ + callsign, + tx: { count: 0, reports: [] }, + rx: { count: 0, reports: [] }, + error: error.message + }); } }); - // ============================================ // SATELLITE TRACKING API // ============================================ diff --git a/src/components/PSKReporterPanel.jsx b/src/components/PSKReporterPanel.jsx index c53451a..2bbb0a7 100644 --- a/src/components/PSKReporterPanel.jsx +++ b/src/components/PSKReporterPanel.jsx @@ -147,13 +147,21 @@ const PSKReporterPanel = ({ callsign, onShowOnMap }) => {
- {loading && reports.length === 0 ? ( + {error ? ( +
+
⚠️ PSKReporter temporarily unavailable
+
Will retry automatically
+
+ ) : loading && reports.length === 0 ? (
Loading...
) : reports.length === 0 ? (
No {activeTab === 'tx' ? 'reception reports' : 'stations heard'} in the last {timeWindow} minutes +
+ (Make sure you're transmitting digital modes like FT8) +
) : ( <> diff --git a/src/hooks/usePSKReporter.js b/src/hooks/usePSKReporter.js index 650be65..f685ee0 100644 --- a/src/hooks/usePSKReporter.js +++ b/src/hooks/usePSKReporter.js @@ -1,28 +1,55 @@ /** * usePSKReporter Hook * Fetches PSKReporter data showing where your signal is being received - * and what stations you're hearing + * + * Uses HTTP API with server-side caching to respect PSKReporter rate limits. + * For real-time updates, connect directly to mqtt.pskreporter.info:1886 (wss) + * Topic: pskr/filter/v2/+/+/YOURCALL/# */ import { useState, useEffect, useCallback } from 'react'; +// Convert grid square to lat/lon +function gridToLatLon(grid) { + if (!grid || grid.length < 4) return null; + + const g = grid.toUpperCase(); + const lon = (g.charCodeAt(0) - 65) * 20 - 180; + const lat = (g.charCodeAt(1) - 65) * 10 - 90; + const lonMin = parseInt(g[2]) * 2; + const latMin = parseInt(g[3]) * 1; + + let finalLon = lon + lonMin + 1; + let finalLat = lat + latMin + 0.5; + + if (grid.length >= 6) { + const lonSec = (g.charCodeAt(4) - 65) * (2/24); + const latSec = (g.charCodeAt(5) - 65) * (1/24); + finalLon = lon + lonMin + lonSec + (1/24); + finalLat = lat + latMin + latSec + (0.5/24); + } + + return { lat: finalLat, lon: finalLon }; +} + export const usePSKReporter = (callsign, options = {}) => { const { minutes = 15, // Time window in minutes (default 15) - direction = 'both', // 'tx' (being heard), 'rx' (hearing), or 'both' enabled = true, // Enable/disable fetching - refreshInterval = 120000 // Refresh every 2 minutes + refreshInterval = 300000, // Refresh every 5 minutes (PSKReporter friendly) + maxSpots = 100 // Max spots to display } = options; - const [txData, setTxData] = useState({ count: 0, reports: [] }); - const [rxData, setRxData] = useState({ count: 0, reports: [] }); + const [txReports, setTxReports] = useState([]); + const [rxReports, setRxReports] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); + const [rateLimited, setRateLimited] = useState(false); const [lastUpdate, setLastUpdate] = useState(null); const fetchData = useCallback(async () => { if (!callsign || callsign === 'N0CALL' || !enabled) { - setTxData({ count: 0, reports: [] }); - setRxData({ count: 0, reports: [] }); + setTxReports([]); + setRxReports([]); setLoading(false); return; } @@ -30,38 +57,56 @@ export const usePSKReporter = (callsign, options = {}) => { try { setError(null); - if (direction === 'both') { - // Fetch combined endpoint - const response = await fetch(`/api/pskreporter/${encodeURIComponent(callsign)}?minutes=${minutes}`); - if (response.ok) { - const data = await response.json(); - setTxData(data.tx || { count: 0, reports: [] }); - setRxData(data.rx || { count: 0, reports: [] }); - } - } else if (direction === 'tx') { - // Fetch only TX (where am I being heard) - const response = await fetch(`/api/pskreporter/tx/${encodeURIComponent(callsign)}?minutes=${minutes}`); - if (response.ok) { - const data = await response.json(); - setTxData(data); - } - } else if (direction === 'rx') { - // Fetch only RX (what am I hearing) - const response = await fetch(`/api/pskreporter/rx/${encodeURIComponent(callsign)}?minutes=${minutes}`); - if (response.ok) { - const data = await response.json(); - setRxData(data); + // Fetch combined endpoint from our server (handles caching) + const response = await fetch(`/api/pskreporter/${encodeURIComponent(callsign)}?minutes=${minutes}`); + + if (response.ok) { + const data = await response.json(); + + // Process TX reports (where I'm being heard) + const txData = data.tx?.reports || []; + const processedTx = txData + .map(r => ({ + ...r, + // Ensure we have location data + lat: r.lat || (r.receiverGrid ? gridToLatLon(r.receiverGrid)?.lat : null), + lon: r.lon || (r.receiverGrid ? gridToLatLon(r.receiverGrid)?.lon : null), + age: r.age || Math.floor((Date.now() - r.timestamp) / 60000) + })) + .filter(r => r.lat && r.lon) + .slice(0, maxSpots); + + // Process RX reports (what I'm hearing) + const rxData = data.rx?.reports || []; + const processedRx = rxData + .map(r => ({ + ...r, + lat: r.lat || (r.senderGrid ? gridToLatLon(r.senderGrid)?.lat : null), + lon: r.lon || (r.senderGrid ? gridToLatLon(r.senderGrid)?.lon : null), + age: r.age || Math.floor((Date.now() - r.timestamp) / 60000) + })) + .filter(r => r.lat && r.lon) + .slice(0, maxSpots); + + setTxReports(processedTx); + setRxReports(processedRx); + setRateLimited(data.tx?.rateLimited || data.rx?.rateLimited || false); + setLastUpdate(new Date()); + + // Check for errors in response + if (data.error || data.tx?.error || data.rx?.error) { + setError(data.error || data.tx?.error || data.rx?.error); } + } else { + throw new Error(`HTTP ${response.status}`); } - - setLastUpdate(new Date()); } catch (err) { console.error('PSKReporter fetch error:', err); setError(err.message); } finally { setLoading(false); } - }, [callsign, minutes, direction, enabled]); + }, [callsign, minutes, enabled, maxSpots]); useEffect(() => { fetchData(); @@ -72,46 +117,30 @@ export const usePSKReporter = (callsign, options = {}) => { } }, [fetchData, enabled, refreshInterval]); - // Computed values - const txReports = txData.reports || []; - const rxReports = rxData.reports || []; - - // Get unique bands from TX reports - const txBands = [...new Set(txReports.map(r => r.band))].filter(b => b !== 'Unknown'); - - // Get unique modes from TX reports - const txModes = [...new Set(txReports.map(r => r.mode))]; + // Computed stats + const txBands = [...new Set(txReports.map(r => r.band))].filter(b => b && b !== 'Unknown'); + const txModes = [...new Set(txReports.map(r => r.mode))].filter(Boolean); - // Stats const stats = { - txCount: txData.count || 0, - rxCount: rxData.count || 0, + txCount: txReports.length, + rxCount: rxReports.length, txBands, txModes, - furthestTx: txReports.length > 0 - ? txReports.reduce((max, r) => r.distance > (max?.distance || 0) ? r : max, null) - : null, bestSnr: txReports.length > 0 ? txReports.reduce((max, r) => (r.snr || -99) > (max?.snr || -99) ? r : max, null) : null }; return { - // TX data - where is my signal being heard txReports, - txCount: txData.count || 0, - - // RX data - what am I hearing + txCount: txReports.length, rxReports, - rxCount: rxData.count || 0, - - // Combined + rxCount: rxReports.length, stats, loading, error, + rateLimited, lastUpdate, - - // Manual refresh refresh: fetchData }; };