2018-10-01 07:37:52 -04:00
|
|
|
#include "LotWUsers.hpp"
|
|
|
|
|
|
|
|
#include <future>
|
2019-01-01 11:19:01 -05:00
|
|
|
#include <chrono>
|
2018-10-01 07:37:52 -04:00
|
|
|
|
|
|
|
#include <QHash>
|
|
|
|
#include <QString>
|
|
|
|
#include <QDate>
|
|
|
|
#include <QFile>
|
|
|
|
#include <QTextStream>
|
|
|
|
#include <QDir>
|
2018-10-01 16:19:21 -04:00
|
|
|
#include <QFileInfo>
|
|
|
|
#include <QPointer>
|
|
|
|
#include <QSaveFile>
|
|
|
|
#include <QUrl>
|
|
|
|
#include <QNetworkAccessManager>
|
|
|
|
#include <QNetworkReply>
|
2018-10-01 07:37:52 -04:00
|
|
|
#include <QDebug>
|
|
|
|
|
|
|
|
#include "pimpl_impl.hpp"
|
|
|
|
|
|
|
|
#include "moc_LotWUsers.cpp"
|
|
|
|
|
|
|
|
namespace
|
|
|
|
{
|
|
|
|
// Dictionary mapping call sign to date of last upload to LotW
|
|
|
|
using dictionary = QHash<QString, QDate>;
|
2018-10-01 16:19:21 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
class LotWUsers::impl final
|
|
|
|
: public QObject
|
|
|
|
{
|
|
|
|
Q_OBJECT
|
|
|
|
|
|
|
|
public:
|
2018-10-01 17:43:13 -04:00
|
|
|
impl (LotWUsers * self, QNetworkAccessManager * network_manager)
|
2018-10-01 16:19:21 -04:00
|
|
|
: self_ {self}
|
|
|
|
, network_manager_ {network_manager}
|
|
|
|
, url_valid_ {false}
|
|
|
|
, redirect_count_ {0}
|
2018-10-16 19:26:04 -04:00
|
|
|
, age_constraint_ {365}
|
2018-10-01 16:19:21 -04:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2019-03-02 08:22:42 -05:00
|
|
|
void load (QString const& url, bool fetch, bool forced_fetch)
|
2018-10-01 16:19:21 -04:00
|
|
|
{
|
|
|
|
abort (); // abort any active download
|
2019-03-02 08:22:42 -05:00
|
|
|
auto csv_file_name = csv_file_.fileName ();
|
|
|
|
auto exists = QFileInfo::exists (csv_file_name);
|
|
|
|
if (fetch && (!exists || forced_fetch))
|
2018-10-01 16:19:21 -04:00
|
|
|
{
|
2018-10-16 19:26:04 -04:00
|
|
|
current_url_.setUrl (url);
|
2018-10-17 17:31:10 -04:00
|
|
|
if (current_url_.isValid () && !QSslSocket::supportsSsl ())
|
|
|
|
{
|
|
|
|
current_url_.setScheme ("http");
|
|
|
|
}
|
2018-10-01 16:19:21 -04:00
|
|
|
redirect_count_ = 0;
|
|
|
|
download (current_url_);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2019-03-02 08:22:42 -05:00
|
|
|
if (exists)
|
|
|
|
{
|
|
|
|
// load the database asynchronously
|
|
|
|
future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_name);
|
|
|
|
}
|
2018-10-01 16:19:21 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void download (QUrl url)
|
|
|
|
{
|
2018-10-16 19:26:04 -04:00
|
|
|
if (QNetworkAccessManager::Accessible != network_manager_->networkAccessible ())
|
|
|
|
{
|
|
|
|
// try and recover network access for QNAM
|
|
|
|
network_manager_->setNetworkAccessible (QNetworkAccessManager::Accessible);
|
|
|
|
}
|
2018-10-01 16:19:21 -04:00
|
|
|
|
|
|
|
QNetworkRequest request {url};
|
|
|
|
request.setRawHeader ("User-Agent", "WSJT LotW User Downloader");
|
|
|
|
request.setOriginatingObject (this);
|
|
|
|
|
|
|
|
// this blocks for a second or two the first time it is used on
|
|
|
|
// Windows - annoying
|
|
|
|
if (!url_valid_)
|
|
|
|
{
|
|
|
|
reply_ = network_manager_->head (request);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
reply_ = network_manager_->get (request);
|
|
|
|
}
|
|
|
|
|
|
|
|
connect (reply_.data (), &QNetworkReply::finished, this, &LotWUsers::impl::reply_finished);
|
|
|
|
connect (reply_.data (), &QNetworkReply::readyRead, this, &LotWUsers::impl::store);
|
|
|
|
}
|
|
|
|
|
|
|
|
void reply_finished ()
|
|
|
|
{
|
2018-10-16 19:26:04 -04:00
|
|
|
if (!reply_)
|
|
|
|
{
|
|
|
|
Q_EMIT self_->load_finished ();
|
|
|
|
return; // we probably deleted it in an earlier call
|
|
|
|
}
|
2018-10-01 16:19:21 -04:00
|
|
|
QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()};
|
|
|
|
if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ())
|
|
|
|
{
|
2018-10-17 17:31:10 -04:00
|
|
|
if ("https" == redirect_url.scheme () && !QSslSocket::supportsSsl ())
|
|
|
|
{
|
|
|
|
Q_EMIT self_->LotW_users_error (tr ("Network Error - SSL/TLS support not installed, cannot fetch:\n\'%1\'")
|
|
|
|
.arg (redirect_url.toDisplayString ()));
|
|
|
|
url_valid_ = false; // reset
|
|
|
|
Q_EMIT self_->load_finished ();
|
|
|
|
}
|
|
|
|
else if (++redirect_count_ < 10) // maintain sanity
|
2018-10-01 16:19:21 -04:00
|
|
|
{
|
|
|
|
// follow redirect
|
|
|
|
download (reply_->url ().resolved (redirect_url));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
Q_EMIT self_->LotW_users_error (tr ("Network Error - Too many redirects:\n\'%1\'")
|
|
|
|
.arg (redirect_url.toDisplayString ()));
|
|
|
|
url_valid_ = false; // reset
|
2018-10-16 19:26:04 -04:00
|
|
|
Q_EMIT self_->load_finished ();
|
2018-10-01 16:19:21 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else if (reply_->error () != QNetworkReply::NoError)
|
|
|
|
{
|
|
|
|
csv_file_.cancelWriting ();
|
|
|
|
csv_file_.commit ();
|
|
|
|
url_valid_ = false; // reset
|
|
|
|
// report errors that are not due to abort
|
|
|
|
if (QNetworkReply::OperationCanceledError != reply_->error ())
|
|
|
|
{
|
|
|
|
Q_EMIT self_->LotW_users_error (tr ("Network Error:\n%1")
|
|
|
|
.arg (reply_->errorString ()));
|
|
|
|
}
|
2018-10-16 19:26:04 -04:00
|
|
|
Q_EMIT self_->load_finished ();
|
2018-10-01 16:19:21 -04:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (url_valid_ && !csv_file_.commit ())
|
|
|
|
{
|
|
|
|
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot commit changes to:\n\"%1\"")
|
|
|
|
.arg (csv_file_.fileName ()));
|
|
|
|
url_valid_ = false; // reset
|
2018-10-16 19:26:04 -04:00
|
|
|
Q_EMIT self_->load_finished ();
|
2018-10-01 16:19:21 -04:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (!url_valid_)
|
|
|
|
{
|
|
|
|
// now get the body content
|
|
|
|
url_valid_ = true;
|
|
|
|
download (reply_->url ().resolved (redirect_url));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
url_valid_ = false; // reset
|
|
|
|
// load the database asynchronously
|
|
|
|
future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_.fileName ());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (reply_ && reply_->isFinished ())
|
|
|
|
{
|
|
|
|
reply_->deleteLater ();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void store ()
|
|
|
|
{
|
|
|
|
if (url_valid_)
|
|
|
|
{
|
|
|
|
if (!csv_file_.isOpen ())
|
|
|
|
{
|
|
|
|
// create temporary file in the final location
|
|
|
|
if (!csv_file_.open (QSaveFile::WriteOnly))
|
|
|
|
{
|
|
|
|
abort ();
|
|
|
|
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot open file:\n\"%1\"\nError(%2): %3")
|
|
|
|
.arg (csv_file_.fileName ())
|
|
|
|
.arg (csv_file_.error ())
|
|
|
|
.arg (csv_file_.errorString ()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (csv_file_.write (reply_->read (reply_->bytesAvailable ())) < 0)
|
|
|
|
{
|
|
|
|
abort ();
|
|
|
|
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot write to file:\n\"%1\"\nError(%2): %3")
|
|
|
|
.arg (csv_file_.fileName ())
|
|
|
|
.arg (csv_file_.error ())
|
|
|
|
.arg (csv_file_.errorString ()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void abort ()
|
|
|
|
{
|
|
|
|
if (reply_ && reply_->isRunning ())
|
|
|
|
{
|
|
|
|
reply_->abort ();
|
|
|
|
}
|
|
|
|
}
|
2018-10-01 07:37:52 -04:00
|
|
|
|
|
|
|
// Load the database from the given file name
|
|
|
|
//
|
|
|
|
// Expects the file to be in CSV format with no header with one
|
|
|
|
// record per line. Record fields are call sign followed by upload
|
|
|
|
// date in yyyy-MM-dd format followed by upload time (ignored)
|
2018-10-01 16:19:21 -04:00
|
|
|
dictionary load_dictionary (QString const& lotw_csv_file)
|
2018-10-01 07:37:52 -04:00
|
|
|
{
|
|
|
|
dictionary result;
|
2018-10-01 16:19:21 -04:00
|
|
|
QFile f {lotw_csv_file};
|
2018-10-01 07:37:52 -04:00
|
|
|
if (f.open (QFile::ReadOnly | QFile::Text))
|
|
|
|
{
|
|
|
|
QTextStream s {&f};
|
|
|
|
for (auto l = s.readLine (); !l.isNull (); l = s.readLine ())
|
|
|
|
{
|
|
|
|
auto pos = l.indexOf (',');
|
|
|
|
result[l.left (pos)] = QDate::fromString (l.mid (pos + 1, l.indexOf (',', pos + 1) - pos - 1), "yyyy-MM-dd");
|
|
|
|
}
|
2018-10-04 11:32:38 -04:00
|
|
|
// qDebug () << "LotW User Data Loaded";
|
2018-10-01 07:37:52 -04:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2018-10-03 14:18:54 -04:00
|
|
|
throw std::runtime_error {QObject::tr ("Failed to open LotW users CSV file: '%1'").arg (f.fileName ()).toStdString ()};
|
2018-10-01 07:37:52 -04:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2018-10-01 16:19:21 -04:00
|
|
|
LotWUsers * self_;
|
|
|
|
QNetworkAccessManager * network_manager_;
|
|
|
|
QSaveFile csv_file_;
|
|
|
|
bool url_valid_;
|
|
|
|
QUrl current_url_; // may be a redirect
|
|
|
|
int redirect_count_;
|
|
|
|
QPointer<QNetworkReply> reply_;
|
2018-10-01 07:37:52 -04:00
|
|
|
std::future<dictionary> future_load_;
|
|
|
|
dictionary last_uploaded_;
|
2018-10-16 19:26:04 -04:00
|
|
|
qint64 age_constraint_; // days
|
2018-10-01 07:37:52 -04:00
|
|
|
};
|
|
|
|
|
2018-10-01 16:19:21 -04:00
|
|
|
#include "LotWUsers.moc"
|
|
|
|
|
2018-10-01 17:43:13 -04:00
|
|
|
LotWUsers::LotWUsers (QNetworkAccessManager * network_manager, QObject * parent)
|
2018-10-01 07:37:52 -04:00
|
|
|
: QObject {parent}
|
2018-10-01 17:43:13 -04:00
|
|
|
, m_ {this, network_manager}
|
2018-10-01 07:37:52 -04:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
LotWUsers::~LotWUsers ()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2018-10-16 19:26:04 -04:00
|
|
|
void LotWUsers::set_local_file_path (QString const& path)
|
|
|
|
{
|
|
|
|
m_->csv_file_.setFileName (path);
|
|
|
|
}
|
|
|
|
|
2019-03-02 08:22:42 -05:00
|
|
|
void LotWUsers::load (QString const& url, bool fetch, bool force_download)
|
2018-10-16 19:26:04 -04:00
|
|
|
{
|
2019-03-02 08:22:42 -05:00
|
|
|
m_->load (url, fetch, force_download);
|
2018-10-16 19:26:04 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
void LotWUsers::set_age_constraint (qint64 uploaded_since_days)
|
2018-10-01 16:19:21 -04:00
|
|
|
{
|
2018-10-16 19:26:04 -04:00
|
|
|
m_->age_constraint_ = uploaded_since_days;
|
2018-10-01 16:19:21 -04:00
|
|
|
}
|
|
|
|
|
2018-10-16 19:26:04 -04:00
|
|
|
bool LotWUsers::user (QString const& call) const
|
2018-10-01 07:37:52 -04:00
|
|
|
{
|
2019-01-01 11:19:01 -05:00
|
|
|
// check if a pending asynchronous load is ready
|
|
|
|
if (m_->future_load_.valid ()
|
|
|
|
&& std::future_status::ready == m_->future_load_.wait_for (std::chrono::seconds {0}))
|
2018-10-01 07:37:52 -04:00
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
|
|
|
// wait for the load to finish if necessary
|
|
|
|
const_cast<dictionary&> (m_->last_uploaded_) = const_cast<std::future<dictionary>&> (m_->future_load_).get ();
|
|
|
|
}
|
|
|
|
catch (std::exception const& e)
|
|
|
|
{
|
|
|
|
Q_EMIT LotW_users_error (e.what ());
|
|
|
|
}
|
2018-10-16 19:26:04 -04:00
|
|
|
Q_EMIT load_finished ();
|
2018-10-01 07:37:52 -04:00
|
|
|
}
|
2019-01-01 11:19:01 -05:00
|
|
|
if (m_->last_uploaded_.size ())
|
2018-10-01 07:37:52 -04:00
|
|
|
{
|
2019-01-01 11:19:01 -05:00
|
|
|
auto p = m_->last_uploaded_.constFind (call);
|
|
|
|
if (p != m_->last_uploaded_.end ())
|
|
|
|
{
|
|
|
|
return p.value ().daysTo (QDate::currentDate ()) <= m_->age_constraint_;
|
|
|
|
}
|
2018-10-01 07:37:52 -04:00
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|