w4kek/WSJT-X
1
0
Fork 0
mirror of https://github.com/saitohirga/WSJT-X.git synced 2025-05-23 18:02:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

show status of LotW file download; new button to download of CTY.DAT, show version of CTY.DAT; genericize download file code;

Browse Source
This commit is contained in:
Brian Moran 2023-03-15 20:42:03 -07:00
parent eebababece
commit b812ac2786
26 changed files with 471 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -222,6 +222,7 @@ set (wsjt_qt_CXXSRCS
widgets/DoubleClickablePushButton.cpp
widgets/DoubleClickableRadioButton.cpp
Network/LotWUsers.cpp
Network/FileDownload.cpp
models/DecodeHighlightingModel.cpp
widgets/DecodeHighlightingListView.cpp
models/FoxLog.cpp

51
Configuration.cpp
View File

@ -201,6 +201,7 @@
#include "models/DecodeHighlightingModel.hpp"
#include "logbook/logbook.h"
#include "widgets/LazyFillComboBox.hpp"
#include "Network/FileDownload.hpp"
#include "ui_Configuration.h"
#include "moc_Configuration.cpp"
@ -564,6 +565,8 @@ private:
Q_SLOT void on_add_macro_line_edit_editingFinished ();
Q_SLOT void delete_macro ();
void delete_selected_macros (QModelIndexList);
void after_CTY_downloaded();
void set_CTY_DAT_version(QString const& version);
Q_SLOT void on_udp_server_line_edit_textChanged (QString const&);
Q_SLOT void on_udp_server_line_edit_editingFinished ();
Q_SLOT void on_save_path_select_push_button_clicked (bool);
@ -574,7 +577,9 @@ private:
Q_SLOT void handle_transceiver_failure (QString const& reason);
Q_SLOT void on_reset_highlighting_to_defaults_push_button_clicked (bool);
Q_SLOT void on_rescan_log_push_button_clicked (bool);
Q_SLOT void on_CTY_download_button_clicked (bool);
Q_SLOT void on_LotW_CSV_fetch_push_button_clicked (bool);
Q_SLOT void on_cbx2ToneSpacing_clicked(bool);
Q_SLOT void on_cbx4ToneSpacing_clicked(bool);
Q_SLOT void on_prompt_to_log_check_box_clicked(bool);
@ -746,7 +751,7 @@ private:
QAudioDeviceInfo next_audio_output_device_;
AudioDevice::Channel audio_output_channel_;
AudioDevice::Channel next_audio_output_channel_;
FileDownload cty_download;
friend class Configuration;
};
@ -859,6 +864,11 @@ bool Configuration::highlight_73 () const {return m_->highlight_73_;}
bool Configuration::highlight_DXcall () const {return m_->highlight_DXcall_;}
bool Configuration::highlight_DXgrid () const {return m_->highlight_DXgrid_;}
void Configuration::set_CTY_DAT_version(QString const& version)
{
m_->set_CTY_DAT_version(version);
}
void Configuration::set_calibration (CalibrationParams params)
{
m_->calibration_ = params;
@ -1174,8 +1184,13 @@ Configuration::impl::impl (Configuration * self, QNetworkAccessManager * network
// set up LoTW users CSV file fetching
connect (&lotw_users_, &LotWUsers::load_finished, [this] () {
ui_->LotW_CSV_fetch_push_button->setEnabled (true);
});
ui_->LotW_CSV_fetch_push_button->setEnabled (true);
});
connect(&lotw_users_, &LotWUsers::progress, [this] (QString const& msg) {
ui_->LotW_CSV_status_label->setText(msg);
});
lotw_users_.set_local_file_path (writeable_data_dir_.absoluteFilePath ("lotw-user-activity.csv"));
//
@ -2402,9 +2417,37 @@ void Configuration::impl::on_reset_highlighting_to_defaults_push_button_clicked
void Configuration::impl::on_rescan_log_push_button_clicked (bool /*clicked*/)
{
if (logbook_) logbook_->rescan ();
if (logbook_) {
logbook_->rescan ();
}
}
void Configuration::impl::on_CTY_download_button_clicked (bool /*clicked*/)
{
ui_->CTY_download_button->setEnabled (false); // disable button until download is complete
QDir dataPath {QStandardPaths::writableLocation (QStandardPaths::AppDataLocation)};
cty_download.configure(network_manager_,
"http://www.country-files.com/bigcty/cty.dat",
dataPath.absoluteFilePath("cty.dat"),
"WSJT-X CTY Downloader");
// set up LoTW users CSV file fetching
connect (&cty_download, &FileDownload::complete, this, &Configuration::impl::after_CTY_downloaded, Qt::UniqueConnection);
cty_download.start_download();
}
void Configuration::impl::set_CTY_DAT_version(QString const& version)
{
ui_->CTY_file_label->setText(QString{"CTY File Version: %1"}.arg(version));
}
void Configuration::impl::after_CTY_downloaded ()
{
ui_->CTY_download_button->setEnabled (true);
if (logbook_) {
logbook_->rescan ();
ui_->CTY_file_label->setText(QString{"CTY File Version: %1"}.arg(logbook_->cty_version()));
}
}
void Configuration::impl::on_LotW_CSV_fetch_push_button_clicked (bool /*checked*/)
{
lotw_users_.load (ui_->LotW_CSV_URL_line_edit->text (), true, true);

2
Configuration.hpp
View File

@ -243,6 +243,8 @@ public:
// Close down connection to rig.
void transceiver_offline ();
void set_CTY_DAT_version(QString const& version);
// Set transceiver frequency in Hertz.
Q_SLOT void transceiver_frequency (Frequency);

129
Configuration.ui
View File

@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>588</width>
<height>642</height>
<width>684</width>
<height>662</height>
</rect>
</property>
<property name="windowTitle">
@ -2407,8 +2407,14 @@ Right click for insert and delete options.</string>
<property name="title">
<string>Logbook of the World User Validation</string>
</property>
<layout class="QFormLayout" name="formLayout_18">
<item row="1" column="0">
<layout class="QGridLayout" name="gridLayout_13">
<property name="topMargin">
<number>4</number>
</property>
<property name="bottomMargin">
<number>4</number>
</property>
<item row="0" column="0">
<widget class="QLabel" name="label_15">
<property name="text">
<string>Users CSV file URL:</string>
@ -2418,7 +2424,7 @@ Right click for insert and delete options.</string>
</property>
</widget>
</item>
<item row="1" column="1">
<item row="0" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_20">
<item>
<widget class="QLineEdit" name="LotW_CSV_URL_line_edit">
@ -2445,7 +2451,7 @@ Right click for insert and delete options.</string>
</item>
</layout>
</item>
<item row="2" column="0">
<item row="1" column="0">
<widget class="QLabel" name="label_14">
<property name="text">
<string>Age of last upload less than:</string>
@ -2455,27 +2461,57 @@ Right click for insert and delete options.</string>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QSpinBox" name="LotW_days_since_upload_spin_box">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Adjust this spin box to set the age threshold of LotW user's last upload date that is accepted as a current LotW user.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="accessibleName">
<string>Days since last upload</string>
</property>
<property name="suffix">
<string> days</string>
</property>
<property name="minimum">
<number>0</number>
</property>
<property name="maximum">
<number>9999</number>
</property>
<property name="value">
<number>365</number>
</property>
</widget>
<item row="1" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_23">
<item>
<widget class="QSpinBox" name="LotW_days_since_upload_spin_box">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Adjust this spin box to set the age threshold of LotW user's last upload date that is accepted as a current LotW user.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="accessibleName">
<string>Days since last upload</string>
</property>
<property name="suffix">
<string> days</string>
</property>
<property name="minimum">
<number>0</number>
</property>
<property name="maximum">
<number>9999</number>
</property>
<property name="value">
<number>365</number>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_13">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QLabel" name="LotW_CSV_status_label">
<property name="minimumSize">
<size>
<width>240</width>
<height>0</height>
</size>
</property>
<property name="text">
<string/>
</property>
</widget>
</item>
</layout>
</item>
</layout>
</widget>
@ -2493,6 +2529,35 @@ Right click for insert and delete options.</string>
</property>
</spacer>
</item>
<item>
<widget class="QGroupBox" name="groupBox_8">
<property name="title">
<string>CTY File Download</string>
</property>
<layout class="QGridLayout" name="gridLayout_17">
<property name="topMargin">
<number>4</number>
</property>
<property name="bottomMargin">
<number>4</number>
</property>
<item row="0" column="0">
<widget class="QLabel" name="CTY_file_label">
<property name="text">
<string>CTY File Version: </string>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="QPushButton" name="CTY_download_button">
<property name="text">
<string>Download Latest CTY.dat</string>
</property>
</widget>
</item>
</layout>
</widget>
</item>
<item>
<spacer name="verticalSpacer_6">
<property name="orientation">
@ -2501,7 +2566,7 @@ Right click for insert and delete options.</string>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
<height>20</height>
</size>
</property>
</spacer>
@ -3336,13 +3401,13 @@ Right click for insert and delete options.</string>
</connection>
</connections>
<buttongroups>
<buttongroup name="CAT_stop_bits_button_group"/>
<buttongroup name="CAT_handshake_button_group"/>
<buttongroup name="TX_mode_button_group"/>
<buttongroup name="CAT_data_bits_button_group"/>
<buttongroup name="PTT_method_button_group"/>
<buttongroup name="split_mode_button_group"/>
<buttongroup name="CAT_stop_bits_button_group"/>
<buttongroup name="special_op_activity_button_group"/>
<buttongroup name="split_mode_button_group"/>
<buttongroup name="TX_mode_button_group"/>
<buttongroup name="CAT_handshake_button_group"/>
<buttongroup name="TX_audio_source_button_group"/>
</buttongroups>
</ui>

189
Network/FileDownload.cpp
View File

@ -3,6 +3,8 @@
#include <QCoreApplication>
#include <QUrl>
#include <QNetworkRequest>
#include <QtNetwork/QNetworkAccessManager>
#include <QtNetwork/QNetworkReply>
#include <QFileInfo>
#include <QDir>
#include <QTemporaryFile>
@ -11,7 +13,8 @@
FileDownload::FileDownload() : QObject(nullptr)
{
redirect_count_ = 0;
url_valid_ = false;
}
FileDownload::~FileDownload()
@ -20,14 +23,17 @@ FileDownload::~FileDownload()
void FileDownload::errorOccurred(QNetworkReply::NetworkError code)
{
LOG_INFO(QString{"DOWNLOAD: errorOccurred %1 -> %2"}.arg(code).arg(reply_->errorString()));
//LOG_INFO(QString{ "DOWNLOAD: server returned %1"}.arg(reply_->))
LOG_INFO(QString{"FileDownload [%1]: errorOccurred %2 -> %3"}.arg(user_agent_).arg(code).arg(reply_->errorString()));
Q_EMIT error (reply_->errorString ());
delete tmpfile_;
}
void FileDownload::configure(const QString &source_url, const QString &destination_path)
void FileDownload::configure(QNetworkAccessManager *network_manager, const QString &source_url, const QString &destination_path, const QString &user_agent)
{
manager_ = network_manager;
source_url_ = source_url;
destination_filename_ = destination_path;
user_agent_ = user_agent;
}
void FileDownload::store()
@ -35,17 +41,89 @@ void FileDownload::store()
if (tmpfile_->isOpen())
tmpfile_->write (reply_->read (reply_->bytesAvailable ()));
else
LOG_INFO(QString{ "DOWNLOAD: tmpfile is not open"});
LOG_INFO(QString{ "FileDownload [%1]: tmpfile is not open"}.arg(user_agent_));
}
void FileDownload::replyComplete()
{
auto is_error = reply_->error ();
LOG_INFO(QString{"DOWNLOAD: reply complete %1"}.arg(is_error));
QFileInfo destination_file(destination_filename_);
QString const tmpfile_path = destination_file.absolutePath();
QDir tmpdir_(tmpfile_path);
LOG_DEBUG(QString{ "FileDownload [%1]: replyComplete"}.arg(user_agent_));
if (!reply_)
{
Q_EMIT load_finished ();
return; // we probably deleted it in an earlier call
}
QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()};
if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ())
{
if ("https" == redirect_url.scheme () && !QSslSocket::supportsSsl ())
{
Q_EMIT download_error (tr ("Network Error - SSL/TLS support not installed, cannot fetch:\n\'%1\'")
.arg (redirect_url.toDisplayString ()));
url_valid_ = false; // reset
Q_EMIT load_finished ();
}
else if (++redirect_count_ < 10) // maintain sanity
{
// follow redirect
download (reply_->url ().resolved (redirect_url));
}
else
{
Q_EMIT download_error (tr ("Network Error - Too many redirects:\n\'%1\'")
.arg (redirect_url.toDisplayString ()));
url_valid_ = false; // reset
Q_EMIT load_finished ();
}
}
else if (reply_->error () != QNetworkReply::NoError)
{
tmpfile_->close();
delete tmpfile_;
url_valid_ = false; // reset
// report errors that are not due to abort
if (QNetworkReply::OperationCanceledError != reply_->error ())
{
Q_EMIT download_error (tr ("Network Error:\n%1")
.arg (reply_->errorString ()));
}
Q_EMIT load_finished ();
}
else
{
if (!url_valid_)
{
// now get the body content
url_valid_ = true;
download (reply_->url ().resolved (redirect_url));
}
else // the body has completed. Save it.
{
url_valid_ = false; // reset
// load the database asynchronously
// future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_.fileName ());
LOG_INFO(QString{ "FileDownload [%1]: complete. tempfile path is %2"}.arg(user_agent_).arg(tmpfile_->fileName()));
// move the file to the destination
tmpdir_.remove(destination_filename_+".old"); // get rid of previous version
tmpdir_.rename(destination_filename_, destination_filename_+".old");
tmpdir_.rename(tmpfile_->fileName(), destination_filename_);
LOG_INFO(QString{ "FileDownload [%1]: moved tempfile %2 to %3"}.arg(user_agent_).arg(tmpfile_->fileName()).arg(destination_filename_));
tmpfile_->close();
delete tmpfile_;
emit complete(destination_filename_);
}
}
if (reply_ && reply_->isFinished ())
{
reply_->deleteLater ();
}
}
void FileDownload::downloadComplete(QNetworkReply *data)
@ -53,75 +131,84 @@ void FileDownload::downloadComplete(QNetworkReply *data)
// make a temp file in the same place as the file we're downloading. Needs to be on the same
// filesystem as where we eventually want to 'mv' it.
QUrl r = request_->url();
LOG_INFO(QString{"DOWNLOAD: finished download %1 -> %2 (%3)"}.arg(source_url_).arg(destination_filename_).arg(r.url()));
LOG_INFO(QString{ "DOWNLOAD: tempfile path is %1"}.arg(tmpfile_->fileName()));
tmpfile_->close();
LOG_INFO(QString{"DOWNLOAD: moving file to %2"}.arg(destination_filename_));
QUrl r = request_.url();
LOG_INFO(QString{"FileDownload [%1]: finished %2 of %3 -> %4 (%5)"}.arg(user_agent_).arg(data->operation()).arg(source_url_).arg(destination_filename_).arg(r.url()));
#ifdef DEBUG_FILEDOWNLOAD
LOG_INFO("Request Headers:");
Q_FOREACH (const QByteArray& hdr, request_->rawHeaderList()) {
LOG_INFO(QString{ "%1 -> %2"}.arg(QString(hdr)).arg(QString(request_->rawHeader(hdr))));
}
Q_FOREACH (const QByteArray& hdr, request_.rawHeaderList()) {
LOG_INFO(QString{ "%1 -> %2"}.arg(QString(hdr)).arg(QString(request_.rawHeader(hdr))));
}
LOG_INFO("Response Headers:");
Q_FOREACH (const QByteArray& hdr, reply_->rawHeaderList()) {
LOG_INFO(QString{ "%1 -> %2"}.arg(QString(hdr)).arg(QString(reply_->rawHeader(hdr))));
}
// move the file to the destination
tmpdir_->remove(destination_filename_+".old"); // get rid of previous version
tmpdir_->rename(destination_filename_, destination_filename_+".old");
tmpdir_->rename(tmpfile_->fileName(), destination_filename_);
emit complete(destination_filename_);
#endif
data->deleteLater();
}
void FileDownload::download()
void FileDownload::start_download()
{
//QUrl url = QUrl(source_url_);
url_valid_ = false;
download(QUrl(source_url_));
}
manager_ = new QNetworkAccessManager(this);
void FileDownload::download(QUrl qurl)
{
request_.setUrl(qurl);
// request_ = new QNetworkRequest("https://www.country-files.com/bigcty/cty.dat");
request_ = new QNetworkRequest(QUrl(source_url_));
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
if (QNetworkAccessManager::Accessible != manager_->networkAccessible ())
{
// try and recover network access for QNAM
manager_->setNetworkAccessible (QNetworkAccessManager::Accessible);
}
#endif
LOG_INFO(QString{"DOWNLOAD: starting download %1 -> %2"}.arg(source_url_).arg(destination_filename_));
LOG_INFO(QString{"FileDownload [%1]: Starting download of %2 to %3"}.arg(user_agent_).arg(source_url_).arg(destination_filename_));
request_->setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
//request_->setHeader( QNetworkRequest::ContentTypeHeader, "some/type" );
request_->setRawHeader("Accept", "*/*");
request_->setRawHeader ("User-Agent", "WSJT-X CTY Downloader");
request_.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
request_.setRawHeader("Accept", "*/*");
request_.setRawHeader ("User-Agent", user_agent_.toLocal8Bit()); // Must have a UA for some sites, like country-files
reply_ = manager_->get(*request_);
if (!url_valid_)
{
reply_ = manager_->head(request_);
}
else
{
reply_ = manager_->get (request_);
}
reply_->setReadBufferSize(0);
QObject::connect(manager_, &QNetworkAccessManager::finished, this, &FileDownload::downloadComplete, Qt::UniqueConnection);
QObject::connect(reply_, &QNetworkReply::downloadProgress, this, &FileDownload::downloadProgress, Qt::UniqueConnection);
QObject::connect(reply_, &QNetworkReply::finished, this,&FileDownload::replyComplete, Qt::UniqueConnection);
int http_code = reply_->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
QObject::connect(reply_, &QNetworkReply::errorOccurred,this,&FileDownload::errorOccurred, Qt::UniqueConnection);
QObject::connect (reply_, &QNetworkReply::readyRead, this, &FileDownload::store, Qt::UniqueConnection);
QObject::connect(manager_, &QNetworkAccessManager::finished, this, &FileDownload::downloadComplete);
QObject::connect(reply_, &QNetworkReply::downloadProgress, this, &FileDownload::downloadProgress);
QObject::connect(reply_, &QNetworkReply::finished, this,&FileDownload::replyComplete);
QObject::connect(reply_, &QNetworkReply::errorOccurred,this,&FileDownload::errorOccurred);
QObject::connect (reply_, &QNetworkReply::finished, this, &FileDownload::replyComplete);
QObject::connect (reply_, &QNetworkReply::readyRead, this, &FileDownload::store);
QFileInfo tmpfi(destination_filename_);
QString const tmpfile_path = tmpfi.absolutePath();
tmpdir_ = new QDir(tmpfile_path);
tmpfile_ = new QTemporaryFile(tmpfile_path+"/big.cty.XXXXXX");
QFileInfo destination_file(destination_filename_);
QString const tmpfile_base = destination_file.fileName();
QString const tmpfile_path = destination_file.absolutePath();
tmpfile_ = new QTemporaryFile(tmpfile_path+QDir::separator()+tmpfile_base+".XXXXXX");
if (!tmpfile_->open())
{
LOG_INFO(QString{"DOWNLOAD: Unable to open the temporary file based on %1"}.arg(tmpfile_path));
LOG_INFO(QString{"FileDownload [%1]: Unable to open the temporary file based on %2"}.arg(user_agent_).arg(tmpfile_path));
return;
}
LOG_INFO(QString{"DOWNLOAD: let's go %1"}.arg(http_code));
}
void FileDownload::downloadProgress(qint64 received, qint64 total)
{
LOG_INFO(QString{"DOWNLOAD: Progress %1 from %2, total %3, so far %4"}.arg(destination_filename_).arg(source_url_).arg(total).arg(received));
//qDebug() << received << total;
LOG_DEBUG(QString{"FileDownload: [%1] Progress %2 from %3, total %4, so far %5"}.arg(user_agent_).arg(destination_filename_).arg(source_url_).arg(total).arg(received));
Q_EMIT progress(QString{"%4 bytes downloaded"}.arg(received));
}
void FileDownload::abort ()
{
if (reply_ && reply_->isRunning ())
{
reply_->abort ();
}
}

22
Network/FileDownload.hpp
View File

@ -3,6 +3,7 @@
#include <QObject>
#include <QString>
#include <QPointer>
#include <QtNetwork/QNetworkAccessManager>
#include <QtNetwork/QNetworkReply>
#include <QTemporaryFile>
@ -14,22 +15,31 @@ public:
explicit FileDownload();
~FileDownload();
void configure(const QString& source_url, const QString& destination_filename);
void configure(QNetworkAccessManager *network_manager, const QString& source_url, const QString& destination_filename, const QString& user_agent);
private:
QNetworkAccessManager *manager_;
QString source_url_;
QString destination_filename_;
QNetworkReply *reply_;
QNetworkRequest *request_;
QTemporaryFile *tmpfile_;
QDir *tmpdir_;
QString user_agent_;
QPointer<QNetworkReply> reply_;
QNetworkRequest request_;
QPointer<QTemporaryFile> tmpfile_;
bool url_valid_;
int redirect_count_;
signals:
void complete(QString filename);
void progress(QString filename);
void load_finished() const;
void download_error (QString const& reason) const;
void error(QString const& reason) const;
public slots:
void download();
void start_download();
void download(QUrl url);
void store();
void abort();
void downloadComplete(QNetworkReply* data);
void downloadProgress(qint64 recieved, qint64 total);
void errorOccurred(QNetworkReply::NetworkError code);

182
Network/LotWUsers.cpp
View File

@ -18,7 +18,7 @@
#include <QDebug>
#include "qt_helpers.hpp"
#include "Logger.hpp"
#include "FileDownload.hpp"
#include "pimpl_impl.hpp"
#include "moc_LotWUsers.cpp"
@ -41,6 +41,7 @@ public:
, url_valid_ {false}
, redirect_count_ {0}
, age_constraint_ {365}
, connected_ {false}
{
}
@ -50,14 +51,36 @@ public:
auto csv_file_name = csv_file_.fileName ();
auto exists = QFileInfo::exists (csv_file_name);
if (fetch && (!exists || forced_fetch))
{
current_url_.setUrl(url);
if (current_url_.isValid() && !QSslSocket::supportsSsl())
{
current_url_.setUrl (url);
if (current_url_.isValid () && !QSslSocket::supportsSsl ())
{
current_url_.setScheme ("http");
}
redirect_count_ = 0;
download (current_url_);
current_url_.setScheme("http");
}
redirect_count_ = 0;
Q_EMIT self_->progress (QString("Starting download from %1").arg(url));
lotw_downloader_.configure(network_manager_,
url,
csv_file_name,
"WSJT-X LotW User Downloader");
if (!connected_)
{
connect(&lotw_downloader_, &FileDownload::complete, [this, csv_file_name] {
LOG_INFO(QString{"LotWUsers: Loading LotW file %1"}.arg(csv_file_name));
future_load_ = std::async(std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_name);
});
connect(&lotw_downloader_, &FileDownload::error, [this] (QString const& msg) {
LOG_INFO(QString{"LotWUsers: Error downloading LotW file: %1"}.arg(msg));
Q_EMIT self_->LotW_users_error (msg);
});
connect( &lotw_downloader_, &FileDownload::progress, [this] (QString const& msg) {
Q_EMIT self_->progress (msg);
});
connected_ = true;
}
lotw_downloader_.start_download();
}
else
{
@ -69,143 +92,9 @@ public:
}
}
void download (QUrl url)
{
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
if (QNetworkAccessManager::Accessible != network_manager_->networkAccessible ())
{
// try and recover network access for QNAM
network_manager_->setNetworkAccessible (QNetworkAccessManager::Accessible);
}
#endif
LOG_INFO(QString("Download..."));
QNetworkRequest request {url};
request.setRawHeader ("User-Agent", "WSJT LotW User Downloader");
request.setOriginatingObject (this);
// this blocks for a second or two the first time it is used on
// Windows - annoying
if (!url_valid_)
{
reply_ = network_manager_->head (request);
}
else
{
reply_ = network_manager_->get (request);
}
connect (reply_.data (), &QNetworkReply::finished, this, &LotWUsers::impl::reply_finished);
connect (reply_.data (), &QNetworkReply::readyRead, this, &LotWUsers::impl::store);
}
void reply_finished ()
{
LOG_INFO(QString("Finished..."));
if (!reply_)
{
Q_EMIT self_->load_finished ();
return; // we probably deleted it in an earlier call
}
QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()};
if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ())
{
if ("https" == redirect_url.scheme () && !QSslSocket::supportsSsl ())
{
Q_EMIT self_->LotW_users_error (tr ("Network Error - SSL/TLS support not installed, cannot fetch:\n\'%1\'")
.arg (redirect_url.toDisplayString ()));
url_valid_ = false; // reset
Q_EMIT self_->load_finished ();
}
else if (++redirect_count_ < 10) // maintain sanity
{
// follow redirect
download (reply_->url ().resolved (redirect_url));
}
else
{
Q_EMIT self_->LotW_users_error (tr ("Network Error - Too many redirects:\n\'%1\'")
.arg (redirect_url.toDisplayString ()));
url_valid_ = false; // reset
Q_EMIT self_->load_finished ();
}
}
else if (reply_->error () != QNetworkReply::NoError)
{
csv_file_.cancelWriting ();
csv_file_.commit ();
url_valid_ = false; // reset
// report errors that are not due to abort
if (QNetworkReply::OperationCanceledError != reply_->error ())
{
Q_EMIT self_->LotW_users_error (tr ("Network Error:\n%1")
.arg (reply_->errorString ()));
}
Q_EMIT self_->load_finished ();
}
else
{
if (url_valid_ && !csv_file_.commit ())
{
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot commit changes to:\n\"%1\"")
.arg (csv_file_.fileName ()));
url_valid_ = false; // reset
Q_EMIT self_->load_finished ();
}
else
{
if (!url_valid_)
{
// now get the body content
url_valid_ = true;
download (reply_->url ().resolved (redirect_url));
}
else
{
url_valid_ = false; // reset
// load the database asynchronously
future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_.fileName ());
}
}
}
if (reply_ && reply_->isFinished ())
{
reply_->deleteLater ();
}
}
void store ()
{
if (url_valid_)
{
if (!csv_file_.isOpen ())
{
// create temporary file in the final location
if (!csv_file_.open (QSaveFile::WriteOnly))
{
abort ();
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot open file:\n\"%1\"\nError(%2): %3")
.arg (csv_file_.fileName ())
.arg (csv_file_.error ())
.arg (csv_file_.errorString ()));
}
}
if (csv_file_.write (reply_->read (reply_->bytesAvailable ())) < 0)
{
abort ();
Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot write to file:\n\"%1\"\nError(%2): %3")
.arg (csv_file_.fileName ())
.arg (csv_file_.error ())
.arg (csv_file_.errorString ()));
}
}
}
void abort ()
{
if (reply_ && reply_->isRunning ())
{
reply_->abort ();
}
lotw_downloader_.abort();
}
// Load the database from the given file name
@ -225,12 +114,14 @@ public:
auto pos = l.indexOf (',');
result[l.left (pos)] = QDate::fromString (l.mid (pos + 1, l.indexOf (',', pos + 1) - pos - 1), "yyyy-MM-dd");
}
// qDebug () << "LotW User Data Loaded";
}
else
{
throw std::runtime_error {QObject::tr ("Failed to open LotW users CSV file: '%1'").arg (f.fileName ()).toStdString ()};
}
LOG_INFO(QString{"LotWUsers: Loaded %1 records from %2"}.arg(result.size()).arg(lotw_csv_file));
Q_EMIT self_->progress (QString{"Loaded %1 records from LotW."}.arg(result.size()));
Q_EMIT self_->load_finished();
return result;
}
@ -244,6 +135,8 @@ public:
std::future<dictionary> future_load_;
dictionary last_uploaded_;
qint64 age_constraint_; // days
FileDownload lotw_downloader_;
bool connected_;
};
#include "LotWUsers.moc"
@ -252,6 +145,7 @@ LotWUsers::LotWUsers (QNetworkAccessManager * network_manager, QObject * parent)
: QObject {parent}
, m_ {this, network_manager}
{
}
LotWUsers::~LotWUsers ()

1
Network/LotWUsers.hpp
View File

@ -31,6 +31,7 @@ public:
bool user (QString const& call) const;
Q_SIGNAL void LotW_users_error (QString const& reason) const;
Q_SIGNAL void progress (QString const& reason) const;
Q_SIGNAL void load_finished () const;
private:

151
logbook/AD1CCty.cpp
View File

@ -19,6 +19,7 @@
#include "Configuration.hpp"
#include "Radio.hpp"
#include "pimpl_impl.hpp"
#include "Logger.hpp"
#include "moc_AD1CCty.cpp"
@ -163,6 +164,9 @@ public:
{
}
QString get_cty_path(const Configuration *configuration);
void load_cty(QFile &file);
entity_by_id::iterator lookup_entity (QString call, prefix const& p) const
{
call = call.toUpper ();
@ -228,6 +232,7 @@ public:
Configuration const * configuration_;
QString path_;
QString cty_version_;
entities_type entities_;
prefixes_type prefixes_;
};
@ -314,6 +319,72 @@ char const * AD1CCty::continent (Continent c)
}
}
QString AD1CCty::impl::get_cty_path(Configuration const * configuration)
{
QDir dataPath {QStandardPaths::writableLocation (QStandardPaths::AppDataLocation)};
auto path = dataPath.exists (file_name)
? dataPath.absoluteFilePath (file_name) // user override
: configuration->data_dir ().absoluteFilePath (file_name); // or original
return path;
}
void AD1CCty::impl::load_cty(QFile &file)
{
int entity_id = 0;
int line_number{0};
entities_.clear();
prefixes_.clear();
QTextStream in{&file};
while (!in.atEnd())
{
auto const &entity_line = in.readLine();
++line_number;
if (!in.atEnd())
{
auto const &entity_parts = entity_line.split(':');
if (entity_parts.size() >= 8)
{
auto primary_prefix = entity_parts[7].trimmed();
bool WAE_only{false};
if (primary_prefix.startsWith('*'))
{
primary_prefix = primary_prefix.mid(1);
WAE_only = true;
}
bool ok1, ok2, ok3, ok4, ok5;
entities_.emplace(++entity_id, entity_parts[0].trimmed(), WAE_only, entity_parts[1].trimmed().toInt(&ok1),
entity_parts[2].trimmed().toInt(&ok2), continent(entity_parts[3].trimmed()),
entity_parts[4].trimmed().toFloat(&ok3), entity_parts[5].trimmed().toFloat(&ok4),
static_cast<int> (entity_parts[6].trimmed().toFloat(&ok5) * 60 * 60), primary_prefix);
if (!(ok1 && ok2 && ok3 && ok4 && ok5))
{
throw std::domain_error{"Invalid number in cty.dat line " + boost::lexical_cast<std::string>(line_number)};
}
QString line;
QString detail;
do
{
in.readLineInto(&line);
++line_number;
} while (detail += line, !detail.endsWith(';'));
for (auto prefix: detail.left(detail.size() - 1).split(','))
{
prefix = prefix.trimmed();
bool exact{false};
if (prefix.startsWith('='))
{
prefix = prefix.mid(1);
exact = true;
}
prefixes_.emplace(prefix, exact, entity_id);
}
}
}
}
}
AD1CCty::AD1CCty (Configuration const * configuration)
: m_ {configuration}
{
@ -321,69 +392,23 @@ AD1CCty::AD1CCty (Configuration const * configuration)
// TODO: G4WJS - consider doing the following asynchronously to
// speed up startup. Not urgent as it takes less than 0.5s on a Core
// i7 reading BIG CTY.DAT.
QDir dataPath {QStandardPaths::writableLocation (QStandardPaths::DataLocation)};
m_->path_ = dataPath.exists (file_name)
? dataPath.absoluteFilePath (file_name) // user override
: configuration->data_dir ().absoluteFilePath (file_name); // or original
AD1CCty::reload (configuration);
}
void AD1CCty::reload(Configuration const * configuration)
{
m_->path_ = m_->impl::get_cty_path(configuration);
QFile file {m_->path_};
LOG_INFO(QString{"Loading CTY.DAT from %1"}.arg (m_->path_));
if (file.open (QFile::ReadOnly))
{
int entity_id = 0;
int line_number {0};
QTextStream in {&file};
while (!in.atEnd ())
{
auto const& entity_line = in.readLine ();
++line_number;
if (!in.atEnd ())
{
auto const& entity_parts = entity_line.split (':');
if (entity_parts.size () >= 8)
{
auto primary_prefix = entity_parts[7].trimmed ();
bool WAE_only {false};
if (primary_prefix.startsWith ('*'))
{
primary_prefix = primary_prefix.mid (1);
WAE_only = true;
}
bool ok1, ok2, ok3, ok4, ok5;
m_->entities_.emplace (++entity_id
, entity_parts[0].trimmed ()
, WAE_only
, entity_parts[1].trimmed ().toInt (&ok1)
, entity_parts[2].trimmed ().toInt (&ok2)
, continent (entity_parts[3].trimmed ())
, entity_parts[4].trimmed ().toFloat (&ok3)
, entity_parts[5].trimmed ().toFloat (&ok4)
, static_cast<int> (entity_parts[6].trimmed ().toFloat (&ok5) * 60 * 60)
, primary_prefix);
if (!(ok1 && ok2 && ok3 && ok4 && ok5))
{
throw std::domain_error {"Invalid number in cty.dat line " + boost::lexical_cast<std::string> (line_number)};
}
QString line;
QString detail;
do
{
in.readLineInto (&line);
++line_number;
} while (detail += line, !detail.endsWith (';'));
for (auto prefix : detail.left (detail.size () - 1).split (','))
{
prefix = prefix.trimmed ();
bool exact {false};
if (prefix.startsWith ('='))
{
prefix = prefix.mid (1);
exact = true;
}
m_->prefixes_.emplace (prefix, exact, entity_id);
}
}
}
}
}
{
m_->impl::load_cty(file);
m_->cty_version_ = AD1CCty::lookup("VERSION").entity_name;
Q_EMIT cty_loaded(m_->cty_version_);
LOG_INFO(QString{"Loaded CTY.DAT version %1"}.arg (m_->cty_version_));
}
}
AD1CCty::~AD1CCty ()
@ -421,3 +446,7 @@ auto AD1CCty::lookup (QString const& call) const -> Record
}
return Record {};
}
auto AD1CCty::version () const -> QString
{
return m_->cty_version_;
}

3
logbook/AD1CCty.hpp
View File

@ -42,8 +42,11 @@ public:
};
explicit AD1CCty (Configuration const *);
void reload(Configuration const * configuration);
~AD1CCty ();
Record lookup (QString const& call) const;
QString version () const;
Q_SIGNAL void cty_loaded (QString const& version) const;
private:
class impl;

16
logbook/WorkedBefore.cpp
View File

@ -23,6 +23,7 @@
#include <QDateTime>
#include "Configuration.hpp"
#include "revision_utils.hpp"
#include "Logger.hpp"
#include "qt_helpers.hpp"
#include "pimpl_impl.hpp"
@ -225,7 +226,7 @@ namespace
{
auto const logFileName = "wsjtx_log.adi";
// Expception class suitable for using with QtConcurrent across
// Exception class suitable for using with QtConcurrent across
// thread boundaries
class LoaderException final
: public QException
@ -374,6 +375,7 @@ public:
void reload ()
{
prefixes_.reload (configuration_);
async_loader_ = QtConcurrent::run (loader, path_, &prefixes_);
loader_watcher_.setFuture (async_loader_);
}
@ -402,11 +404,18 @@ WorkedBefore::WorkedBefore (Configuration const * configuration)
{
error = e.error ();
}
Q_EMIT finished_loading (n, error);
QString cty_ver = m_->prefixes_.version();
LOG_DEBUG(QString{"WorkedBefore::reload: CTY.DAT version %1"}.arg (cty_ver));
Q_EMIT finished_loading (n, cty_ver, error);
});
reload ();
}
QString WorkedBefore::cty_version () const
{
return m_->prefixes_.version ();
}
void WorkedBefore::reload ()
{
m_->reload ();
@ -668,6 +677,7 @@ bool WorkedBefore::CQ_zone_worked (int CQ_zone, QString const& mode, QString con
}
}
bool WorkedBefore::ITU_zone_worked (int ITU_zone, QString const& mode, QString const& band) const
{
if (mode.size ())
@ -699,3 +709,5 @@ bool WorkedBefore::ITU_zone_worked (int ITU_zone, QString const& mode, QString c
}
}
}

3
logbook/WorkedBefore.hpp
View File

@ -36,8 +36,9 @@ public:
bool continent_worked (Continent continent, QString const& mode, QString const& band) const;
bool CQ_zone_worked (int CQ_zone, QString const& mode, QString const& band) const;
bool ITU_zone_worked (int ITU_zone, QString const& mode, QString const& band) const;
QString cty_version () const;
Q_SIGNAL void finished_loading (int worked_before_record_count, QString const& error) const;
Q_SIGNAL void finished_loading (int worked_before_record_count, QString const, QString const& error) const;
private:
class impl;

5
logbook/logbook.cpp
View File

@ -69,6 +69,11 @@ void LogBook::rescan ()
worked_before_.reload ();
}
QString const LogBook::cty_version() const
{
return worked_before_.cty_version();
}
QByteArray LogBook::QSOToADIF (QString const& hisCall, QString const& hisGrid, QString const& mode,
QString const& rptSent, QString const& rptRcvd, QDateTime const& dateTimeOn,
QDateTime const& dateTimeOff, QString const& band, QString const& comments,

4
logbook/logbook.h
View File

@ -46,7 +46,9 @@ public:
QString const& m_myGrid, QString const& m_txPower, QString const& operator_call,
QString const& xSent, QString const& xRcvd, QString const& propmode);
Q_SIGNAL void finished_loading (int worked_before_record_count, QString const& error) const;
QString const cty_version() const;
Q_SIGNAL void finished_loading (int worked_before_record_count, QString const cty_version, QString const& error) const;
CabrilloLog * contest_log ();
Multiplier const * multiplier () const;

2
translations/wsjtx_ca.ts
View File

@ -3684,7 +3684,7 @@ La llista es pot mantenir a la configuració (F2).</translation>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation>Log ADIF escanejat, %1 funcionava abans de la creació de registres</translation>
</message>
<message>

2
translations/wsjtx_da.ts
View File

@ -3917,7 +3917,7 @@ listen. Makro listen kan også ændfres i Inderstillinger (F2).</translation>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation>Scannet ADIF log, %1 worked B4 oprettede poster</translation>
</message>
<message>

2
translations/wsjtx_en.ts
View File

@ -3637,7 +3637,7 @@ list. The list can be maintained in Settings (F2).</source>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation type="unfinished"></translation>
</message>
<message>

2
translations/wsjtx_en_GB.ts
View File

@ -3637,7 +3637,7 @@ list. The list can be maintained in Settings (F2).</source>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created</source>
<translation type="unfinished"></translation>
</message>
<message>

2
translations/wsjtx_es.ts
View File

@ -4237,7 +4237,7 @@ predefinida. La lista se puede modificar en &quot;Ajustes&quot; (F2).</translati
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translatorcomment>Log ADIF escaneado, %1 funcionaba antes de la creación de registros</translatorcomment>
<translation>Log ADIF escaneado, %1 registros trabajados B4 creados</translation>
</message>

2
translations/wsjtx_it.ts
View File

@ -4112,7 +4112,7 @@ elenco. L&apos;elenco può essere gestito in Impostazioni (F2).</translation>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="527"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation>Log ADIF scansionato,%1 ha funzionato prima della creazione dei record</translation>
</message>
<message>

2
translations/wsjtx_ja.ts
View File

@ -3869,7 +3869,7 @@ ENTERを押してテキストを登録リストに追加.
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation>ADIFログ検索. %1交信済み記録作成しました</translation>
</message>
<message>

2
translations/wsjtx_ru.ts
View File

@ -3690,7 +3690,7 @@ list. The list can be maintained in Settings (F2).</source>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation>Просканирован лог ADIF, %1 работал до создания записей</translation>
</message>
<message>

2
translations/wsjtx_zh.ts
View File

@ -3683,7 +3683,7 @@ list. The list can be maintained in Settings (F2).</source>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation>扫描 ADIF 日志, %1 创建曾经通联记录</translation>
</message>
<message>

2
translations/wsjtx_zh_HK.ts
View File

@ -3683,7 +3683,7 @@ list. The list can be maintained in Settings (F2).</source>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created</source>
<translation>掃描 ADIF 紀錄, %1 建立曾經通聯紀錄</translation>
</message>
<message>

2
translations/wsjtx_zh_TW.ts
View File

@ -3714,7 +3714,7 @@ list. The list can be maintained in Settings (F2).</source>
</message>
<message>
<location filename="../widgets/mainwindow.cpp" line="523"/>
<source>Scanned ADIF log, %1 worked before records created</source>
<source>Scanned ADIF log, %1 worked-before records created. CTY: %2</source>
<translation>掃描 ADIF 紀錄, %1 建立曾經通聯紀錄</translation>
</message>
<message>

5
widgets/mainwindow.cpp
View File

@ -522,14 +522,15 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple,
connect (this, &MainWindow::finished, m_logDlg.data (), &LogQSO::close);
// hook up the log book
connect (&m_logBook, &LogBook::finished_loading, [this] (int record_count, QString const& error) {
connect (&m_logBook, &LogBook::finished_loading, [this] (int record_count, QString cty_version, QString const& error) {
if (error.size ())
{
MessageBox::warning_message (this, tr ("Error Scanning ADIF Log"), error);
}
else
{
showStatusMessage (tr ("Scanned ADIF log, %1 worked before records created").arg (record_count));
m_config.set_CTY_DAT_version(cty_version);
showStatusMessage (tr ("Scanned ADIF log, %1 worked-before records created. CTY: %2. CTY: %2").arg (record_count).arg (cty_version));
}
});