Post FST4W spots to WSPRNet.org

Includes a re-factoring  of the WSPRNet class,  particularly to handle
direct spot posts as well as via  a file from wsprd. Switched from GET
http request method to POST method.

FST4W spots  post the same information  a WSPR spots except  the drift
field is  always zero (FST4W  has no  drift compensation, so  no drift
figure is calculated by the decoder),  and the mode field reflects the
T/R  period in  minutes.  This  means  FST4W-120A will  be similar  to
WSPR-2, an FST4W-900  will be similar to WSPR-15. I  don't see any way
to  view the  mode field  on  either the  new or  old database  format
queries on WSPRnet,  so it is hard  to tell if that  field is actually
stored.
This commit is contained in:
Bill Somerville
2020-07-26 02:58:04 +01:00
parent 63b1b0729a
commit 7566f3548d
4 changed files with 354 additions and 220 deletions
+273 -172
View File
@@ -8,6 +8,8 @@
#include <QTimer>
#include <QFile>
#include <QRegExp>
#include <QRegularExpression>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
@@ -18,215 +20,314 @@
namespace
{
char const * const wsprNetUrl = "http://wsprnet.org/post?";
// char const * const wsprNetUrl = "http://127.0.0.1/post?";
char const * const wsprNetUrl = "http://wsprnet.org/post/";
//char const * const wsprNetUrl = "http://127.0.0.1:5000/post/";
//
// tested with this python REST mock of WSPRNet.org
//
/*
# Mock WSPRNet.org RESTful API
from flask import Flask, request, url_for
from flask_restful import Resource, Api
app = Flask(__name__)
@app.route ('/post/', methods=['GET', 'POST'])
def spot ():
if request.method == 'POST':
print (request.form)
return "1 spot(s) added"
with app.test_request_context ():
print (url_for ('spot'))
*/
// regexp to parse FST4W decodes
QRegularExpression fst4_re {R"(
(?<time>\d{4})
\s+(?<db>[-+]?\d+)
\s+(?<dt>[-+]?\d+\.\d+)
\s+(?<freq>\d+)
\s+`
\s+<?(?<call>[A-Z0-9/]+)>?(?:\s(?<grid>[A-R]{2}[0-9]{2}(?:[A-X]{2})?))?(?:\s+(?<dBm>\d+))?
)", QRegularExpression::ExtendedPatternSyntaxOption};
// regexp to parse wspr_spots.txt from wsprd
//
// 130223 2256 7 -21 -0.3 14.097090 DU1MGA PK04 37 0 40 0
// Date Time Sync dBm DT Freq Msg
// 1 2 3 4 5 6 -------7------ 8 9 10
QRegularExpression wspr_re(R"(^(\d+)\s+(\d+)\s+(\d+)\s+([+-]?\d+)\s+([+-]?\d+\.\d+)\s+(\d+\.\d+)\s+([^ ].*[^ ])\s+([+-]?\d+)\s+([+-]?\d+)\s+([+-]?\d+))");
};
WSPRNet::WSPRNet(QNetworkAccessManager * manager, QObject *parent)
: QObject{parent}
, networkManager {manager}
, uploadTimer {new QTimer {this}}
, m_urlQueueSize {0}
WSPRNet::WSPRNet (QNetworkAccessManager * manager, QObject *parent)
: QObject {parent}
, network_manager_ {manager}
, spots_to_send_ {0}
{
connect(networkManager, SIGNAL(finished(QNetworkReply*)), this, SLOT(networkReply(QNetworkReply*)));
connect( uploadTimer, SIGNAL(timeout()), this, SLOT(work()));
connect (network_manager_, &QNetworkAccessManager::finished, this, &WSPRNet::networkReply);
connect (&upload_timer_, &QTimer::timeout, this, &WSPRNet::work);
}
void WSPRNet::upload(QString const& call, QString const& grid, QString const& rfreq, QString const& tfreq,
QString const& mode, QString const& tpct, QString const& dbm, QString const& version,
QString const& fileName)
void WSPRNet::upload (QString const& call, QString const& grid, QString const& rfreq, QString const& tfreq,
QString const& mode, float TR_period, QString const& tpct, QString const& dbm,
QString const& version, QString const& fileName)
{
m_call = call;
m_grid = grid;
m_rfreq = rfreq;
m_tfreq = tfreq;
m_mode = mode;
m_tpct = tpct;
m_dbm = dbm;
m_vers = version;
m_file = fileName;
m_call = call;
m_grid = grid;
m_rfreq = rfreq;
m_tfreq = tfreq;
m_mode = mode;
TR_period_ = TR_period;
m_tpct = tpct;
m_dbm = dbm;
m_vers = version;
m_file = fileName;
// Open the wsprd.out file
QFile wsprdOutFile(fileName);
if (!wsprdOutFile.open(QIODevice::ReadOnly | QIODevice::Text) ||
wsprdOutFile.size() == 0) {
urlQueue.enqueue( wsprNetUrl + urlEncodeNoSpot());
m_uploadType = 1;
uploadTimer->start(200);
return;
// Open the wsprd.out file
QFile wsprdOutFile (fileName);
if (!wsprdOutFile.open (QIODevice::ReadOnly | QIODevice::Text) || !wsprdOutFile.size ())
{
spot_queue_.enqueue (urlEncodeNoSpot ());
m_uploadType = 1;
}
// Read the contents
while (!wsprdOutFile.atEnd()) {
QHash<QString,QString> query;
if ( decodeLine(wsprdOutFile.readLine(), query) ) {
// Prevent reporting data ouside of the current frequency band
float f = fabs(m_rfreq.toFloat() - query["tqrg"].toFloat());
if (f < 0.0002) {
urlQueue.enqueue( wsprNetUrl + urlEncodeSpot(query));
m_uploadType = 2;
else
{
// Read the contents
while (!wsprdOutFile.atEnd())
{
SpotQueue::value_type query;
if (decodeLine (wsprdOutFile.readLine(), query))
{
// Prevent reporting data ouside of the current frequency band
float f = fabs (m_rfreq.toFloat() - query.queryItemValue ("tqrg", QUrl::FullyDecoded).toFloat());
if (f < 0.0002)
{
spot_queue_.enqueue(urlEncodeSpot (query));
m_uploadType = 2;
}
}
}
}
}
m_urlQueueSize = urlQueue.size();
uploadTimer->start(200);
spots_to_send_ = spot_queue_.size ();
upload_timer_.start (200);
}
void WSPRNet::networkReply(QNetworkReply *reply)
void WSPRNet::post (QString const& call, QString const& grid, QString const& rfreq, QString const& tfreq,
QString const& mode, float TR_period, QString const& tpct, QString const& dbm,
QString const& version, QString const& decode_text)
{
m_call = call;
m_grid = grid;
m_rfreq = rfreq;
m_tfreq = tfreq;
m_mode = mode;
TR_period_ = TR_period;
m_tpct = tpct;
m_dbm = dbm;
m_vers = version;
if (!decode_text.size ())
{
if (!spot_queue_.size ())
{
spot_queue_.enqueue (urlEncodeNoSpot ());
m_uploadType = 1;
}
spots_to_send_ = spot_queue_.size ();
upload_timer_.start (200);
}
else
{
auto const& match = fst4_re.match (decode_text);
if (match.hasMatch ())
{
SpotQueue::value_type query;
// Prevent reporting data ouside of the current frequency band
auto tqrg = match.captured ("freq").toInt ();
if (tqrg >= 1400 && tqrg <= 1600)
{
query.addQueryItem ("function", "wspr");
// use time as at 3/4 of T/R period before current to
// ensure date is in Rx period
auto const& date = QDateTime::currentDateTimeUtc ().addSecs (-TR_period * 3. / 4.).date ();
query.addQueryItem ("date", date.toString ("yyMMdd"));
query.addQueryItem ("time", match.captured ("time"));
query.addQueryItem ("sig", match.captured ("db"));
query.addQueryItem ("dt", match.captured ("dt"));
query.addQueryItem ("tqrg", QString::number (rfreq.toDouble () + (tqrg - 1500) / 1e6, 'f', 6));
query.addQueryItem ("tcall", match.captured ("call"));
query.addQueryItem ("drift", "0");
query.addQueryItem ("tgrid", match.captured ("grid"));
query.addQueryItem ("dbm", match.captured ("dBm"));
spot_queue_.enqueue (urlEncodeSpot (query));
m_uploadType = 2;
}
}
}
}
void WSPRNet::networkReply (QNetworkReply * reply)
{
// check if request was ours
if (m_outstandingRequests.removeOne (reply)) {
if (QNetworkReply::NoError != reply->error ()) {
Q_EMIT uploadStatus (QString {"Error: %1"}.arg (reply->error ()));
// not clearing queue or halting queuing as it may be a transient
// one off request error
}
else {
QString serverResponse = reply->readAll();
if( m_uploadType == 2) {
if (!serverResponse.contains(QRegExp("spot\\(s\\) added"))) {
emit uploadStatus(QString {"Upload Failed: %1"}.arg (serverResponse));
urlQueue.clear();
uploadTimer->stop();
if (m_outstandingRequests.removeOne (reply))
{
if (QNetworkReply::NoError != reply->error ())
{
Q_EMIT uploadStatus (QString {"Error: %1"}.arg (reply->error ()));
// not clearing queue or halting queuing as it may be a
// transient one off request error
}
}
else
{
QString serverResponse = reply->readAll ();
if (m_uploadType == 2)
{
if (!serverResponse.contains(QRegExp("spot\\(s\\) added")))
{
Q_EMIT uploadStatus (QString {"Upload Failed: %1"}.arg (serverResponse));
spot_queue_.clear ();
upload_timer_.stop ();
}
}
if (urlQueue.isEmpty()) {
emit uploadStatus("done");
QFile::remove(m_file);
uploadTimer->stop();
}
if (!spot_queue_.size ())
{
Q_EMIT uploadStatus("done");
QFile f {m_file};
if (f.exists ()) f.remove ();
upload_timer_.stop ();
}
}
qDebug () << QString {"WSPRnet.org %1 outstanding requests"}.arg (m_outstandingRequests.size ());
// delete request object instance on return to the event loop otherwise it is leaked
reply->deleteLater ();
}
}
bool WSPRNet::decodeLine (QString const& line, SpotQueue::value_type& query)
{
auto const& rx_match = wspr_re.match (line);
if (rx_match.hasMatch ()) {
int msgType = 0;
QString msg = rx_match.captured (7);
QString call, grid, dbm;
QRegularExpression msgRx;
// Check for Message Type 1
msgRx.setPattern(R"(^([A-Z0-9]{3,6})\s+([A-R]{2}\d{2})\s+(\d+))");
auto match = msgRx.match (msg);
if (match.hasMatch ()) {
msgType = 1;
call = match.captured (1);
grid = match.captured (2);
dbm = match.captured (3);
}
qDebug () << QString {"WSPRnet.org %1 outstanding requests"}.arg (m_outstandingRequests.size ());
// Check for Message Type 2
msgRx.setPattern(R"(^([A-Z0-9/]+)\s+(\d+))");
match = msgRx.match (msg);
if (match.hasMatch ()) {
msgType = 2;
call = match.captured (1);
grid = "";
dbm = match.captured (2);
}
// delete request object instance on return to the event loop otherwise it is leaked
reply->deleteLater ();
// Check for Message Type 3
msgRx.setPattern(R"(^<([A-Z0-9/]+)>\s+([A-R]{2}\d{2}[A-X]{2})\s+(\d+))");
match = msgRx.match (msg);
if (match.hasMatch ()) {
msgType = 3;
call = match.captured (1);
grid = match.captured (2);
dbm = match.captured (3);
}
// Unknown message format
if (!msgType) {
return false;
}
query.addQueryItem ("function", "wspr");
query.addQueryItem ("date", rx_match.captured (1));
query.addQueryItem ("time", rx_match.captured (2));
query.addQueryItem ("sig", rx_match.captured (4));
query.addQueryItem ("dt", rx_match.captured(5));
query.addQueryItem ("drift", rx_match.captured(8));
query.addQueryItem ("tqrg", rx_match.captured(6));
query.addQueryItem ("tcall", call);
query.addQueryItem ("tgrid", grid);
query.addQueryItem ("dbm", dbm);
} else {
return false;
}
return true;
}
bool WSPRNet::decodeLine(QString const& line, QHash<QString,QString> &query)
auto WSPRNet::urlEncodeNoSpot () -> SpotQueue::value_type
{
// 130223 2256 7 -21 -0.3 14.097090 DU1MGA PK04 37 0 40 0
// Date Time Sync dBm DT Freq Msg
// 1 2 3 4 5 6 -------7------ 8 9 10
QRegExp rx("^(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+([+-]?\\d+)\\s+([+-]?\\d+\\.\\d+)\\s+(\\d+\\.\\d+)\\s+(.*)\\s+([+-]?\\d+)\\s+([+-]?\\d+)\\s+([+-]?\\d+)");
if (rx.indexIn(line) != -1) {
int msgType = 0;
QString msg = rx.cap(7);
msg.remove(QRegExp("\\s+$"));
msg.remove(QRegExp("^\\s+"));
QString call, grid, dbm;
QRegExp msgRx;
// Check for Message Type 1
msgRx.setPattern("^([A-Z0-9]{3,6})\\s+([A-Z]{2}\\d{2})\\s+(\\d+)");
if (msgRx.indexIn(msg) != -1) {
msgType = 1;
call = msgRx.cap(1);
grid = msgRx.cap(2);
dbm = msgRx.cap(3);
}
// Check for Message Type 2
msgRx.setPattern("^([A-Z0-9/]+)\\s+(\\d+)");
if (msgRx.indexIn(msg) != -1) {
msgType = 2;
call = msgRx.cap(1);
grid = "";
dbm = msgRx.cap(2);
}
// Check for Message Type 3
msgRx.setPattern("^<([A-Z0-9/]+)>\\s+([A-Z]{2}\\d{2}[A-Z]{2})\\s+(\\d+)");
if (msgRx.indexIn(msg) != -1) {
msgType = 3;
call = msgRx.cap(1);
grid = msgRx.cap(2);
dbm = msgRx.cap(3);
}
// Unknown message format
if (!msgType) {
return false;
}
query["function"] = "wspr";
query["date"] = rx.cap(1);
query["time"] = rx.cap(2);
query["sig"] = rx.cap(4);
query["dt"] = rx.cap(5);
query["drift"] = rx.cap(8);
query["tqrg"] = rx.cap(6);
query["tcall"] = call;
query["tgrid"] = grid;
query["dbm"] = dbm;
} else {
return false;
SpotQueue::value_type query;
query.addQueryItem ("function", "wsprstat");
query.addQueryItem ("rcall", m_call);
query.addQueryItem ("rgrid", m_grid);
query.addQueryItem ("rqrg", m_rfreq);
query.addQueryItem ("tpct", m_tpct);
query.addQueryItem ("tqrg", m_tfreq);
query.addQueryItem ("dbm", m_dbm);
query.addQueryItem ("version", m_vers);
if (m_mode == "WSPR") query.addQueryItem ("mode", "2");
if (m_mode == "WSPR-15") query.addQueryItem ("mode", "15");
if (m_mode == "FST4W")
{
query.addQueryItem ("mode", QString::number (static_cast<int> ((TR_period_ / 60.)+.5)));
}
return true;
return query;;
}
QString WSPRNet::urlEncodeNoSpot()
auto WSPRNet::urlEncodeSpot (SpotQueue::value_type& query) -> SpotQueue::value_type
{
QString queryString;
queryString += "function=wsprstat&";
queryString += "rcall=" + m_call + "&";
queryString += "rgrid=" + m_grid + "&";
queryString += "rqrg=" + m_rfreq + "&";
queryString += "tpct=" + m_tpct + "&";
queryString += "tqrg=" + m_tfreq + "&";
queryString += "dbm=" + m_dbm + "&";
queryString += "version=" + m_vers;
if(m_mode=="WSPR") queryString += "&mode=2";
if(m_mode=="WSPR-15") queryString += "&mode=15";
return queryString;;
}
QString WSPRNet::urlEncodeSpot(QHash<QString,QString> const& query)
{
QString queryString;
queryString += "function=" + query["function"] + "&";
queryString += "rcall=" + m_call + "&";
queryString += "rgrid=" + m_grid + "&";
queryString += "rqrg=" + m_rfreq + "&";
queryString += "date=" + query["date"] + "&";
queryString += "time=" + query["time"] + "&";
queryString += "sig=" + query["sig"] + "&";
queryString += "dt=" + query["dt"] + "&";
queryString += "drift=" + query["drift"] + "&";
queryString += "tqrg=" + query["tqrg"] + "&";
queryString += "tcall=" + query["tcall"] + "&";
queryString += "tgrid=" + query["tgrid"] + "&";
queryString += "dbm=" + query["dbm"] + "&";
queryString += "version=" + m_vers;
if(m_mode=="WSPR") queryString += "&mode=2";
if(m_mode=="WSPR-15") queryString += "&mode=15";
return queryString;
query.addQueryItem ("version", m_vers);
query.addQueryItem ("rcall", m_call);
query.addQueryItem ("rgrid", m_grid);
query.addQueryItem ("rqrg", m_rfreq);
if (m_mode == "WSPR") query.addQueryItem ("mode", "2");
if (m_mode == "WSPR-15") query.addQueryItem ("mode", "15");
if (m_mode == "FST4W")
{
query.addQueryItem ("mode", QString::number (static_cast<int> ((TR_period_ / 60.)+.5)));
}
return query;
}
void WSPRNet::work()
{
if (!urlQueue.isEmpty()) {
if (spots_to_send_ && spot_queue_.size ())
{
#if QT_VERSION < QT_VERSION_CHECK (5, 15, 0)
if (QNetworkAccessManager::Accessible != networkManager->networkAccessible ()) {
// try and recover network access for QNAM
networkManager->setNetworkAccessible (QNetworkAccessManager::Accessible);
}
if (QNetworkAccessManager::Accessible != network_manager_->networkAccessible ()) {
// try and recover network access for QNAM
network_manager_->setNetworkAccessible (QNetworkAccessManager::Accessible);
}
#endif
QUrl url(urlQueue.dequeue());
QNetworkRequest request(url);
m_outstandingRequests << networkManager->get(request);
emit uploadStatus(QString {"Uploading Spot %1/%2"}.arg (m_urlQueueSize - urlQueue.size()).arg (m_urlQueueSize));
} else {
uploadTimer->stop();
}
QNetworkRequest request (QUrl {wsprNetUrl});
request.setHeader (QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded");
auto const& spot = spot_queue_.dequeue ();
m_outstandingRequests << network_manager_->post (request, spot.query (QUrl::FullyEncoded).toUtf8 ());
Q_EMIT uploadStatus(QString {"Uploading Spot %1/%2"}.arg (spots_to_send_ - spot_queue_.size()).arg (spots_to_send_));
}
else
{
upload_timer_.stop ();
}
}
void WSPRNet::abortOutstandingRequests () {
urlQueue.clear ();
spot_queue_.clear ();
for (auto& request : m_outstandingRequests) {
request->abort ();
}
m_urlQueueSize = 0;
}