improve physical structure

This commit is contained in:
sirhc808
2019-07-02 12:45:05 -05:00
parent 27d4471cde
commit 1f57ba5fec
28 changed files with 39 additions and 36 deletions
+297
View File
@@ -0,0 +1,297 @@
#include "LotWUsers.hpp"
#include <future>
#include <chrono>
#include <QHash>
#include <QString>
#include <QDate>
#include <QFile>
#include <QTextStream>
#include <QDir>
#include <QFileInfo>
#include <QPointer>
#include <QSaveFile>
#include <QUrl>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QDebug>
#include "pimpl_impl.hpp"
#include "moc_LotWUsers.cpp"
namespace
{
// Dictionary mapping call sign to date of last upload to LotW
using dictionary = QHash<QString, QDate>;
}
class LotWUsers::impl final
: public QObject
{
Q_OBJECT
public:
impl (LotWUsers * self, QNetworkAccessManager * network_manager)
: self_ {self}
, network_manager_ {network_manager}
, url_valid_ {false}
, redirect_count_ {0}
, age_constraint_ {365}
{
}
void load (QString const& url, bool fetch, bool forced_fetch)
{
abort (); // abort any active download
auto csv_file_name = csv_file_.fileName ();
auto exists = QFileInfo::exists (csv_file_name);
if (fetch && (!exists || forced_fetch))
{
current_url_.setUrl (url);
if (current_url_.isValid () && !QSslSocket::supportsSsl ())
{
current_url_.setScheme ("http");
}
redirect_count_ = 0;
download (current_url_);
}
else
{
if (exists)
{
// load the database asynchronously
future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_name);
}
}
}
void download (QUrl url)
{
if (QNetworkAccessManager::Accessible != network_manager_->networkAccessible ())
{
// try and recover network access for QNAM
network_manager_->setNetworkAccessible (QNetworkAccessManager::Accessible);
}
QNetworkRequest request {url};
request.setRawHeader ("User-Agent", "WSJT LotW User Downloader");
request.setOriginatingObject (this);
// this blocks for a second or two the first time it is used on
// Windows - annoying
if (!url_valid_)
{
reply_ = network_manager_->head (request);
}
else
{
reply_ = network_manager_->get (request);
}
connect (reply_.data (), &QNetworkReply::finished, this, &LotWUsers::impl::reply_finished);
connect (reply_.data (), &QNetworkReply::readyRead, this, &LotWUsers::impl::store);
}
void reply_finished ()
{
if (!reply_)
{
Q_EMIT self_->load_finished ();
return; // we probably deleted it in an earlier call
}
QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()};
if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ())
{
if ("https" == redirect_url.scheme () && !QSslSocket::supportsSsl ())
{
Q_EMIT self_->LotW_users_error (tr ("Network Error - SSL/TLS support not installed, cannot fetch:\n\'%1\'")
.arg (redirect_url.toDisplayString ()));
url_valid_ = false; // reset
Q_EMIT self_->load_finished ();
}
else if (++redirect_count_ < 10) // maintain sanity
{
// follow redirect
download (reply_->url ().resolved (redirect_url));
}
else
{
Q_EMIT self_->LotW_users_error (tr ("Network Error - Too many redirects:\n\'%1\'")
.arg (redirect_url.toDisplayString ()));
url_valid_ = false; // reset
Q_EMIT self_->load_finished ();
}
}
else if (reply_->error () != QNetworkReply::NoError)
{
csv_file_.cancelWriting ();
csv_file_.commit ();
url_valid_ = false; // reset
// report errors that are not due to abort
if (QNetworkReply::OperationCanceledError != reply_->error ())
{
Q_EMIT self_->LotW_users_error (tr ("Network Error:\n%1")
.arg (reply_->errorString ()));
}
Q_EMIT self_->load_finished ();
}
else
{
if (url_valid_ && !csv_file_.commit ())
{
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot commit changes to:\n\"%1\"")
.arg (csv_file_.fileName ()));
url_valid_ = false; // reset
Q_EMIT self_->load_finished ();
}
else
{
if (!url_valid_)
{
// now get the body content
url_valid_ = true;
download (reply_->url ().resolved (redirect_url));
}
else
{
url_valid_ = false; // reset
// load the database asynchronously
future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_.fileName ());
}
}
}
if (reply_ && reply_->isFinished ())
{
reply_->deleteLater ();
}
}
void store ()
{
if (url_valid_)
{
if (!csv_file_.isOpen ())
{
// create temporary file in the final location
if (!csv_file_.open (QSaveFile::WriteOnly))
{
abort ();
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot open file:\n\"%1\"\nError(%2): %3")
.arg (csv_file_.fileName ())
.arg (csv_file_.error ())
.arg (csv_file_.errorString ()));
}
}
if (csv_file_.write (reply_->read (reply_->bytesAvailable ())) < 0)
{
abort ();
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot write to file:\n\"%1\"\nError(%2): %3")
.arg (csv_file_.fileName ())
.arg (csv_file_.error ())
.arg (csv_file_.errorString ()));
}
}
}
void abort ()
{
if (reply_ && reply_->isRunning ())
{
reply_->abort ();
}
}
// Load the database from the given file name
//
// Expects the file to be in CSV format with no header with one
// record per line. Record fields are call sign followed by upload
// date in yyyy-MM-dd format followed by upload time (ignored)
dictionary load_dictionary (QString const& lotw_csv_file)
{
dictionary result;
QFile f {lotw_csv_file};
if (f.open (QFile::ReadOnly | QFile::Text))
{
QTextStream s {&f};
for (auto l = s.readLine (); !l.isNull (); l = s.readLine ())
{
auto pos = l.indexOf (',');
result[l.left (pos)] = QDate::fromString (l.mid (pos + 1, l.indexOf (',', pos + 1) - pos - 1), "yyyy-MM-dd");
}
// qDebug () << "LotW User Data Loaded";
}
else
{
throw std::runtime_error {QObject::tr ("Failed to open LotW users CSV file: '%1'").arg (f.fileName ()).toStdString ()};
}
return result;
}
LotWUsers * self_;
QNetworkAccessManager * network_manager_;
QSaveFile csv_file_;
bool url_valid_;
QUrl current_url_; // may be a redirect
int redirect_count_;
QPointer<QNetworkReply> reply_;
std::future<dictionary> future_load_;
dictionary last_uploaded_;
qint64 age_constraint_; // days
};
#include "LotWUsers.moc"
LotWUsers::LotWUsers (QNetworkAccessManager * network_manager, QObject * parent)
: QObject {parent}
, m_ {this, network_manager}
{
}
LotWUsers::~LotWUsers ()
{
}
void LotWUsers::set_local_file_path (QString const& path)
{
m_->csv_file_.setFileName (path);
}
void LotWUsers::load (QString const& url, bool fetch, bool force_download)
{
m_->load (url, fetch, force_download);
}
void LotWUsers::set_age_constraint (qint64 uploaded_since_days)
{
m_->age_constraint_ = uploaded_since_days;
}
bool LotWUsers::user (QString const& call) const
{
// check if a pending asynchronous load is ready
if (m_->future_load_.valid ()
&& std::future_status::ready == m_->future_load_.wait_for (std::chrono::seconds {0}))
{
try
{
// wait for the load to finish if necessary
const_cast<dictionary&> (m_->last_uploaded_) = const_cast<std::future<dictionary>&> (m_->future_load_).get ();
}
catch (std::exception const& e)
{
Q_EMIT LotW_users_error (e.what ());
}
Q_EMIT load_finished ();
}
if (m_->last_uploaded_.size ())
{
auto p = m_->last_uploaded_.constFind (call);
if (p != m_->last_uploaded_.end ())
{
return p.value ().daysTo (QDate::currentDate ()) <= m_->age_constraint_;
}
}
return false;
}
+41
View File
@@ -0,0 +1,41 @@
#ifndef LOTW_USERS_HPP_
#define LOTW_USERS_HPP_
#include <boost/core/noncopyable.hpp>
#include <QObject>
#include "pimpl_h.hpp"
class QString;
class QDate;
class QNetworkAccessManager;
//
// LotWUsers - Lookup Logbook of the World users
//
class LotWUsers final
: public QObject
{
Q_OBJECT
public:
explicit LotWUsers (QNetworkAccessManager *, QObject * parent = 0);
~LotWUsers ();
void set_local_file_path (QString const&);
Q_SLOT void load (QString const& url, bool fetch = true, bool force_download = false);
Q_SLOT void set_age_constraint (qint64 uploaded_since_days);
// returns true if the specified call sign 'call' has uploaded their
// log to LotW in the last 'age_constraint_days' days
bool user (QString const& call) const;
Q_SIGNAL void LotW_users_error (QString const& reason) const;
Q_SIGNAL void load_finished () const;
private:
class impl;
pimpl<impl> m_;
};
#endif
+230
View File
@@ -0,0 +1,230 @@
// Interface to WSPRnet website
//
// by Edson Pereira - PY2SDR
#include "wsprnet.h"
#include <cmath>
#include <QTimer>
#include <QFile>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QUrl>
#include <QDebug>
#include "moc_wsprnet.cpp"
namespace
{
char const * const wsprNetUrl = "http://wsprnet.org/post?";
// char const * const wsprNetUrl = "http://127.0.0.1/post?";
};
WSPRNet::WSPRNet(QNetworkAccessManager * manager, QObject *parent)
: QObject{parent}
, networkManager {manager}
, uploadTimer {new QTimer {this}}
, m_urlQueueSize {0}
{
connect(networkManager, SIGNAL(finished(QNetworkReply*)), this, SLOT(networkReply(QNetworkReply*)));
connect( uploadTimer, SIGNAL(timeout()), this, SLOT(work()));
}
void WSPRNet::upload(QString const& call, QString const& grid, QString const& rfreq, QString const& tfreq,
QString const& mode, QString const& tpct, QString const& dbm, QString const& version,
QString const& fileName)
{
m_call = call;
m_grid = grid;
m_rfreq = rfreq;
m_tfreq = tfreq;
m_mode = mode;
m_tpct = tpct;
m_dbm = dbm;
m_vers = version;
m_file = fileName;
// Open the wsprd.out file
QFile wsprdOutFile(fileName);
if (!wsprdOutFile.open(QIODevice::ReadOnly | QIODevice::Text) ||
wsprdOutFile.size() == 0) {
urlQueue.enqueue( wsprNetUrl + urlEncodeNoSpot());
m_uploadType = 1;
uploadTimer->start(200);
return;
}
// Read the contents
while (!wsprdOutFile.atEnd()) {
QHash<QString,QString> query;
if ( decodeLine(wsprdOutFile.readLine(), query) ) {
// Prevent reporting data ouside of the current frequency band
float f = fabs(m_rfreq.toFloat() - query["tqrg"].toFloat());
if (f < 0.0002) {
urlQueue.enqueue( wsprNetUrl + urlEncodeSpot(query));
m_uploadType = 2;
}
}
}
m_urlQueueSize = urlQueue.size();
uploadTimer->start(200);
}
void WSPRNet::networkReply(QNetworkReply *reply)
{
// check if request was ours
if (m_outstandingRequests.removeOne (reply)) {
if (QNetworkReply::NoError != reply->error ()) {
Q_EMIT uploadStatus (QString {"Error: %1"}.arg (reply->error ()));
// not clearing queue or halting queuing as it may be a transient
// one off request error
}
else {
QString serverResponse = reply->readAll();
if( m_uploadType == 2) {
if (!serverResponse.contains(QRegExp("spot\\(s\\) added"))) {
emit uploadStatus(QString {"Upload Failed: %1"}.arg (serverResponse));
urlQueue.clear();
uploadTimer->stop();
}
}
if (urlQueue.isEmpty()) {
emit uploadStatus("done");
QFile::remove(m_file);
uploadTimer->stop();
}
}
qDebug () << QString {"WSPRnet.org %1 outstanding requests"}.arg (m_outstandingRequests.size ());
// delete request object instance on return to the event loop otherwise it is leaked
reply->deleteLater ();
}
}
bool WSPRNet::decodeLine(QString const& line, QHash<QString,QString> &query)
{
// 130223 2256 7 -21 -0.3 14.097090 DU1MGA PK04 37 0 40 0
// Date Time Sync dBm DT Freq Msg
// 1 2 3 4 5 6 -------7------ 8 9 10
QRegExp rx("^(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+([+-]?\\d+)\\s+([+-]?\\d+\\.\\d+)\\s+(\\d+\\.\\d+)\\s+(.*)\\s+([+-]?\\d+)\\s+([+-]?\\d+)\\s+([+-]?\\d+)");
if (rx.indexIn(line) != -1) {
int msgType = 0;
QString msg = rx.cap(7);
msg.remove(QRegExp("\\s+$"));
msg.remove(QRegExp("^\\s+"));
QString call, grid, dbm;
QRegExp msgRx;
// Check for Message Type 1
msgRx.setPattern("^([A-Z0-9]{3,6})\\s+([A-Z]{2}\\d{2})\\s+(\\d+)");
if (msgRx.indexIn(msg) != -1) {
msgType = 1;
call = msgRx.cap(1);
grid = msgRx.cap(2);
dbm = msgRx.cap(3);
}
// Check for Message Type 2
msgRx.setPattern("^([A-Z0-9/]+)\\s+(\\d+)");
if (msgRx.indexIn(msg) != -1) {
msgType = 2;
call = msgRx.cap(1);
grid = "";
dbm = msgRx.cap(2);
}
// Check for Message Type 3
msgRx.setPattern("^<([A-Z0-9/]+)>\\s+([A-Z]{2}\\d{2}[A-Z]{2})\\s+(\\d+)");
if (msgRx.indexIn(msg) != -1) {
msgType = 3;
call = msgRx.cap(1);
grid = msgRx.cap(2);
dbm = msgRx.cap(3);
}
// Unknown message format
if (!msgType) {
return false;
}
query["function"] = "wspr";
query["date"] = rx.cap(1);
query["time"] = rx.cap(2);
query["sig"] = rx.cap(4);
query["dt"] = rx.cap(5);
query["drift"] = rx.cap(8);
query["tqrg"] = rx.cap(6);
query["tcall"] = call;
query["tgrid"] = grid;
query["dbm"] = dbm;
} else {
return false;
}
return true;
}
QString WSPRNet::urlEncodeNoSpot()
{
QString queryString;
queryString += "function=wsprstat&";
queryString += "rcall=" + m_call + "&";
queryString += "rgrid=" + m_grid + "&";
queryString += "rqrg=" + m_rfreq + "&";
queryString += "tpct=" + m_tpct + "&";
queryString += "tqrg=" + m_tfreq + "&";
queryString += "dbm=" + m_dbm + "&";
queryString += "version=" + m_vers;
if(m_mode=="WSPR") queryString += "&mode=2";
if(m_mode=="WSPR-15") queryString += "&mode=15";
return queryString;;
}
QString WSPRNet::urlEncodeSpot(QHash<QString,QString> const& query)
{
QString queryString;
queryString += "function=" + query["function"] + "&";
queryString += "rcall=" + m_call + "&";
queryString += "rgrid=" + m_grid + "&";
queryString += "rqrg=" + m_rfreq + "&";
queryString += "date=" + query["date"] + "&";
queryString += "time=" + query["time"] + "&";
queryString += "sig=" + query["sig"] + "&";
queryString += "dt=" + query["dt"] + "&";
queryString += "drift=" + query["drift"] + "&";
queryString += "tqrg=" + query["tqrg"] + "&";
queryString += "tcall=" + query["tcall"] + "&";
queryString += "tgrid=" + query["tgrid"] + "&";
queryString += "dbm=" + query["dbm"] + "&";
queryString += "version=" + m_vers;
if(m_mode=="WSPR") queryString += "&mode=2";
if(m_mode=="WSPR-15") queryString += "&mode=15";
return queryString;
}
void WSPRNet::work()
{
if (!urlQueue.isEmpty()) {
if (QNetworkAccessManager::Accessible != networkManager->networkAccessible ()) {
// try and recover network access for QNAM
networkManager->setNetworkAccessible (QNetworkAccessManager::Accessible);
}
QUrl url(urlQueue.dequeue());
QNetworkRequest request(url);
m_outstandingRequests << networkManager->get(request);
emit uploadStatus(QString {"Uploading Spot %1/%2"}.arg (m_urlQueueSize - urlQueue.size()).arg (m_urlQueueSize));
} else {
uploadTimer->stop();
}
}
void WSPRNet::abortOutstandingRequests () {
urlQueue.clear ();
for (auto& request : m_outstandingRequests) {
request->abort ();
}
m_urlQueueSize = 0;
}
+46
View File
@@ -0,0 +1,46 @@
#ifndef WSPRNET_H
#define WSPRNET_H
#include <QObject>
#include <QString>
#include <QList>
#include <QHash>
#include <QQueue>
class QNetworkAccessManager;
class QTimer;
class QNetworkReply;
class WSPRNet : public QObject
{
Q_OBJECT;
public:
explicit WSPRNet(QNetworkAccessManager *, QObject *parent = nullptr);
void upload(QString const& call, QString const& grid, QString const& rfreq, QString const& tfreq,
QString const& mode, QString const& tpct, QString const& dbm, QString const& version,
QString const& fileName);
static bool decodeLine(QString const& line, QHash<QString,QString> &query);
signals:
void uploadStatus(QString);
public slots:
void networkReply(QNetworkReply *);
void work();
void abortOutstandingRequests ();
private:
QNetworkAccessManager *networkManager;
QList<QNetworkReply *> m_outstandingRequests;
QString m_call, m_grid, m_rfreq, m_tfreq, m_mode, m_tpct, m_dbm, m_vers, m_file;
QQueue<QString> urlQueue;
QTimer *uploadTimer;
int m_urlQueueSize;
int m_uploadType;
QString urlEncodeNoSpot();
QString urlEncodeSpot(QHash<QString,QString> const& spot);
};
#endif // WSPRNET_H