From 0032a00ffe587d456572fca3f3d991cfb6bf112e Mon Sep 17 00:00:00 2001 From: Bill Somerville Date: Mon, 1 Oct 2018 21:19:21 +0100 Subject: [PATCH] Automatic download of ARRL LotW users file at startup if needed --- CMakeLists.txt | 1 - LotWUsers.cpp | 197 ++++++++++++++++++++++++++++++-- LotWUsers.hpp | 5 +- SampleDownloader/RemoteFile.cpp | 142 +++++++++++++++-------- mainwindow.cpp | 2 +- 5 files changed, 285 insertions(+), 62 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 02d34e189..6957eb578 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1428,7 +1428,6 @@ install (FILES install (FILES contrib/Ephemeris/JPLEPH - contrib/lotw-user-activity.csv DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME} #COMPONENT runtime ) diff --git a/LotWUsers.cpp b/LotWUsers.cpp index 0f44e789e..f68d6339e 100644 --- a/LotWUsers.cpp +++ b/LotWUsers.cpp @@ -8,6 +8,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include "Configuration.hpp" @@ -20,16 +26,176 @@ namespace { // Dictionary mapping call sign to date of last upload to LotW using dictionary = QHash; +} + +class LotWUsers::impl final + : public QObject +{ + Q_OBJECT + +public: + impl (LotWUsers * self, QNetworkAccessManager * network_manager, QString const& lotw_csv_file) + : self_ {self} + , network_manager_ {network_manager} + , csv_file_ {lotw_csv_file} + , url_valid_ {false} + , redirect_count_ {0} + { + } + + void load (bool forced_fetch) + { + auto csv_file_name = csv_file_.fileName (); + abort (); // abort any active download + if (!QFileInfo::exists (csv_file_name) || forced_fetch) + { + current_url_.setUrl ("https://lotw.arrl.org/lotw-user-activity.csv"); + redirect_count_ = 0; + download (current_url_); + } + else + { + // load the database asynchronously + future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_name); + } + } + + void download (QUrl url) + { + if (QNetworkAccessManager::Accessible != network_manager_->networkAccessible ()) { + // try and recover network access for QNAM + network_manager_->setNetworkAccessible (QNetworkAccessManager::Accessible); + } + + if (url.isValid () && !QSslSocket::supportsSsl ()) + { + url.setScheme ("http"); + } + QNetworkRequest request {url}; + request.setRawHeader ("User-Agent", "WSJT LotW User Downloader"); + request.setOriginatingObject (this); + + // this blocks for a second or two the first time it is used on + // Windows - annoying + if (!url_valid_) + { + reply_ = network_manager_->head (request); + } + else + { + reply_ = network_manager_->get (request); + } + + connect (reply_.data (), &QNetworkReply::finished, this, &LotWUsers::impl::reply_finished); + connect (reply_.data (), &QNetworkReply::readyRead, this, &LotWUsers::impl::store); + } + + void reply_finished () + { + if (!reply_) return; // we probably deleted it in an + // earlier call + QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()}; + if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ()) + { + if (++redirect_count_ < 10) // maintain sanity + { + // follow redirect + download (reply_->url ().resolved (redirect_url)); + } + else + { + Q_EMIT self_->LotW_users_error (tr ("Network Error - Too many redirects:\n\'%1\'") + .arg (redirect_url.toDisplayString ())); + url_valid_ = false; // reset + } + } + else if (reply_->error () != QNetworkReply::NoError) + { + csv_file_.cancelWriting (); + csv_file_.commit (); + url_valid_ = false; // reset + // report errors that are not due to abort + if (QNetworkReply::OperationCanceledError != reply_->error ()) + { + Q_EMIT self_->LotW_users_error (tr ("Network Error:\n%1") + .arg (reply_->errorString ())); + } + } + else + { + if (url_valid_ && !csv_file_.commit ()) + { + Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot commit changes to:\n\"%1\"") + .arg (csv_file_.fileName ())); + url_valid_ = false; // reset + } + else + { + if (!url_valid_) + { + // now get the body content + url_valid_ = true; + download (reply_->url ().resolved (redirect_url)); + } + else + { + url_valid_ = false; // reset + qDebug () << "LotW Users Data downloaded from" << reply_->url ().toDisplayString (); + // load the database asynchronously + future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_.fileName ()); + } + } + } + if (reply_ && reply_->isFinished ()) + { + reply_->deleteLater (); + } + } + + void store () + { + if (url_valid_) + { + if (!csv_file_.isOpen ()) + { + // create temporary file in the final location + if (!csv_file_.open (QSaveFile::WriteOnly)) + { + abort (); + Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot open file:\n\"%1\"\nError(%2): %3") + .arg (csv_file_.fileName ()) + .arg (csv_file_.error ()) + .arg (csv_file_.errorString ())); + } + } + if (csv_file_.write (reply_->read (reply_->bytesAvailable ())) < 0) + { + abort (); + Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot write to file:\n\"%1\"\nError(%2): %3") + .arg (csv_file_.fileName ()) + .arg (csv_file_.error ()) + .arg (csv_file_.errorString ())); + } + } + } + + void abort () + { + if (reply_ && reply_->isRunning ()) + { + reply_->abort (); + } + } // Load the database from the given file name // // Expects the file to be in CSV format with no header with one // record per line. Record fields are call sign followed by upload // date in yyyy-MM-dd format followed by upload time (ignored) - dictionary load (QString const& lotw_users_file) + dictionary load_dictionary (QString const& lotw_csv_file) { dictionary result; - QFile f {lotw_users_file}; + QFile f {lotw_csv_file}; if (f.open (QFile::ReadOnly | QFile::Text)) { QTextStream s {&f}; @@ -46,26 +212,39 @@ namespace } return result; } -} -class LotWUsers::impl final -{ -public: + LotWUsers * self_; + QNetworkAccessManager * network_manager_; + QSaveFile csv_file_; + bool url_valid_; + QUrl current_url_; // may be a redirect + int redirect_count_; + QPointer reply_; std::future future_load_; dictionary last_uploaded_; }; -LotWUsers::LotWUsers (Configuration const * configuration, QObject * parent) +#include "LotWUsers.moc" + +LotWUsers::LotWUsers (Configuration const * configuration, QNetworkAccessManager * network_manager + , QObject * parent) : QObject {parent} + , m_ {this + , network_manager + , configuration->writeable_data_dir ().absoluteFilePath ("lotw-user-activity.csv")} { - // load the database asynchronously - m_->future_load_ = std::async (std::launch::async, load, configuration->writeable_data_dir ().absoluteFilePath ("lotw-user-activity.csv")); + m_->load (false); } LotWUsers::~LotWUsers () { } +void LotWUsers::download_new_file () +{ + m_->load (true); +} + bool LotWUsers::user (QString const& call, qint64 uploaded_since_days) const { if (m_->future_load_.valid ()) diff --git a/LotWUsers.hpp b/LotWUsers.hpp index 97becd14e..293234bd7 100644 --- a/LotWUsers.hpp +++ b/LotWUsers.hpp @@ -8,6 +8,7 @@ class QString; class QDate; class Configuration; +class QNetworkAccessManager; // // LotWUsers - Lookup Logbook of the World users @@ -18,9 +19,11 @@ class LotWUsers final Q_OBJECT public: - LotWUsers (Configuration const * configuration, QObject * parent = 0); + LotWUsers (Configuration const * configuration, QNetworkAccessManager *, QObject * parent = 0); ~LotWUsers (); + void download_new_file (); + // returns true if the specified call sign 'call' has uploaded their // log to LotW in the last 'uploaded_since_days' days Q_SLOT bool user (QString const& call, qint64 uploaded_since_days) const; diff --git a/SampleDownloader/RemoteFile.cpp b/SampleDownloader/RemoteFile.cpp index 01bfc3f18..e5b4d2cd9 100644 --- a/SampleDownloader/RemoteFile.cpp +++ b/SampleDownloader/RemoteFile.cpp @@ -34,12 +34,15 @@ void RemoteFile::local_file_path (QString const& name) QFile file {local_file_.absoluteFilePath ()}; if (!file.rename (new_file.absoluteFilePath ())) { - listener_->error (tr ("File System Error") - , tr ("Cannot rename file:\n\"%1\"\nto: \"%2\"\nError(%3): %4") - .arg (file.fileName ()) - .arg (new_file.absoluteFilePath ()) - .arg (file.error ()) - .arg (file.errorString ())); + if (listener_) + { + listener_->error (tr ("File System Error") + , tr ("Cannot rename file:\n\"%1\"\nto: \"%2\"\nError(%3): %4") + .arg (file.fileName ()) + .arg (new_file.absoluteFilePath ()) + .arg (file.error ()) + .arg (file.errorString ())); + } } } std::swap (local_file_, new_file); @@ -49,15 +52,18 @@ void RemoteFile::local_file_path (QString const& name) bool RemoteFile::local () const { auto is_local = (reply_ && !reply_->isFinished ()) || local_file_.exists (); - if (is_local) + if (listener_) { - auto size = local_file_.size (); - listener_->download_progress (size, size); - listener_->download_finished (true); - } - else - { - listener_->download_progress (-1, 0); + if (is_local) + { + auto size = local_file_.size (); + listener_->download_progress (size, size); + listener_->download_finished (true); + } + else + { + listener_->download_progress (-1, 0); + } } return is_local; } @@ -92,13 +98,19 @@ bool RemoteFile::sync (QUrl const& url, bool local, bool force) auto path = local_file_.absoluteDir (); if (path.remove (local_file_.fileName ())) { - listener_->download_progress (-1, 0); + if (listener_) + { + listener_->download_progress (-1, 0); + } } else { - listener_->error (tr ("File System Error") - , tr ("Cannot delete file:\n\"%1\"") - .arg (local_file_.absoluteFilePath ())); + if (listener_) + { + listener_->error (tr ("File System Error") + , tr ("Cannot delete file:\n\"%1\"") + .arg (local_file_.absoluteFilePath ())); + } return false; } path.rmpath ("."); @@ -137,10 +149,13 @@ void RemoteFile::download (QUrl url) connect (reply_.data (), &QNetworkReply::readyRead, this, &RemoteFile::store); connect (reply_.data (), &QNetworkReply::downloadProgress , [this] (qint64 bytes_received, qint64 total_bytes) { - // report progress of wanted file - if (is_valid_) + if (listener_) { - listener_->download_progress (bytes_received, total_bytes); + // report progress of wanted file + if (is_valid_) + { + listener_->download_progress (bytes_received, total_bytes); + } } }); } @@ -160,7 +175,7 @@ void RemoteFile::reply_finished () QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()}; if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ()) { - if (listener_->redirect_request (redirect_url)) + if (!listener_ || listener_->redirect_request (redirect_url)) { if (++redirect_count_ < 10) // maintain sanity { @@ -169,19 +184,25 @@ void RemoteFile::reply_finished () } else { - listener_->download_finished (false); - listener_->error (tr ("Network Error") - , tr ("Too many redirects: %1") - .arg (redirect_url.toDisplayString ())); + if (listener_) + { + listener_->download_finished (false); + listener_->error (tr ("Network Error") + , tr ("Too many redirects: %1") + .arg (redirect_url.toDisplayString ())); + } is_valid_ = false; // reset } } else { - listener_->download_finished (false); - listener_->error (tr ("Network Error") - , tr ("Redirect not followed: %1") - .arg (redirect_url.toDisplayString ())); + if (listener_) + { + listener_->download_finished (false); + listener_->error (tr ("Network Error") + , tr ("Redirect not followed: %1") + .arg (redirect_url.toDisplayString ())); + } is_valid_ = false; // reset } } @@ -189,10 +210,13 @@ void RemoteFile::reply_finished () { file_.cancelWriting (); file_.commit (); - listener_->download_finished (false); + if (listener_) + { + listener_->download_finished (false); + } is_valid_ = false; // reset // report errors that are not due to abort - if (QNetworkReply::OperationCanceledError != reply_->error ()) + if (listener_ && QNetworkReply::OperationCanceledError != reply_->error ()) { listener_->error (tr ("Network Error"), reply_->errorString ()); } @@ -202,11 +226,17 @@ void RemoteFile::reply_finished () auto path = QFileInfo {file_.fileName ()}.absoluteDir (); if (is_valid_ && !file_.commit ()) { - listener_->error (tr ("File System Error") - , tr ("Cannot commit changes to:\n\"%1\"") - .arg (file_.fileName ())); + if (listener_) + { + listener_->error (tr ("File System Error") + , tr ("Cannot commit changes to:\n\"%1\"") + .arg (file_.fileName ())); + } path.rmpath ("."); // tidy empty directories - listener_->download_finished (false); + if (listener_) + { + listener_->download_finished (false); + } is_valid_ = false; // reset } else @@ -219,7 +249,10 @@ void RemoteFile::reply_finished () } else { - listener_->download_finished (true); + if (listener_) + { + listener_->download_finished (true); + } is_valid_ = false; // reset } } @@ -243,29 +276,38 @@ void RemoteFile::store () if (!file_.open (QSaveFile::WriteOnly)) { abort (); - listener_->error (tr ("File System Error") - , tr ("Cannot open file:\n\"%1\"\nError(%2): %3") - .arg (path.path ()) - .arg (file_.error ()) - .arg (file_.errorString ())); + if (listener_) + { + listener_->error (tr ("File System Error") + , tr ("Cannot open file:\n\"%1\"\nError(%2): %3") + .arg (path.path ()) + .arg (file_.error ()) + .arg (file_.errorString ())); + } } } else { abort (); - listener_->error (tr ("File System Error") - , tr ("Cannot make path:\n\"%1\"") - .arg (path.path ())); + if (listener_) + { + listener_->error (tr ("File System Error") + , tr ("Cannot make path:\n\"%1\"") + .arg (path.path ())); + } } } if (file_.write (reply_->read (reply_->bytesAvailable ())) < 0) { abort (); - listener_->error (tr ("File System Error") - , tr ("Cannot write to file:\n\"%1\"\nError(%2): %3") - .arg (file_.fileName ()) - .arg (file_.error ()) - .arg (file_.errorString ())); + if (listener_) + { + listener_->error (tr ("File System Error") + , tr ("Cannot write to file:\n\"%1\"\nError(%2): %3") + .arg (file_.fileName ()) + .arg (file_.error ()) + .arg (file_.errorString ())); + } } } } diff --git a/mainwindow.cpp b/mainwindow.cpp index aafa14cc6..f04cbe9cc 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -203,7 +203,7 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple, m_settings {multi_settings->settings ()}, ui(new Ui::MainWindow), m_config {temp_directory, m_settings, this}, - m_lotw_users {&m_config}, + m_lotw_users {&m_config, &m_network_manager}, m_WSPR_band_hopping {m_settings, &m_config, this}, m_WSPR_tx_next {false}, m_rigErrorMessageBox {MessageBox::Critical, tr ("Rig Control Error")