Automatic download of ARRL LotW users file at startup if needed

CMakeLists.txt

@@ -1428,7 +1428,6 @@ install (FILES
 
 install (FILES
   contrib/Ephemeris/JPLEPH
-  contrib/lotw-user-activity.csv
   DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME}
   #COMPONENT runtime
   )

LotWUsers.cpp

@@ -8,6 +8,12 @@
 #include <QFile>
 #include <QTextStream>
 #include <QDir>
+#include <QFileInfo>
+#include <QPointer>
+#include <QSaveFile>
+#include <QUrl>
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
 #include <QDebug>
 
 #include "Configuration.hpp"
@@ -20,16 +26,176 @@ namespace
 {
   // Dictionary mapping call sign to date of last upload to LotW
   using dictionary = QHash<QString, QDate>;
+}
+
+class LotWUsers::impl final
+  : public QObject
+{
+  Q_OBJECT
+
+public:
+  impl (LotWUsers * self, QNetworkAccessManager * network_manager, QString const& lotw_csv_file)
+    : self_ {self}
+    , network_manager_ {network_manager}
+    , csv_file_ {lotw_csv_file}
+    , url_valid_ {false}
+    , redirect_count_ {0}
+  {
+  }
+
+  void load (bool forced_fetch)
+  {
+    auto csv_file_name = csv_file_.fileName ();
+    abort ();                   // abort any active download
+    if (!QFileInfo::exists (csv_file_name) || forced_fetch)
+      {
+        current_url_.setUrl ("https://lotw.arrl.org/lotw-user-activity.csv");
+        redirect_count_ = 0;
+        download (current_url_);
+      }
+    else
+      {
+        // load the database asynchronously
+        future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_name);
+      }
+  }
+
+  void download (QUrl url)
+  {
+    if (QNetworkAccessManager::Accessible != network_manager_->networkAccessible ()) {
+      // try and recover network access for QNAM
+      network_manager_->setNetworkAccessible (QNetworkAccessManager::Accessible);
+    }
+
+    if (url.isValid () && !QSslSocket::supportsSsl ())
+      {
+        url.setScheme ("http");
+      }
+    QNetworkRequest request {url};
+    request.setRawHeader ("User-Agent", "WSJT LotW User Downloader");
+    request.setOriginatingObject (this);
+
+    // this blocks for a second or two the first time it is used on
+    // Windows - annoying
+    if (!url_valid_)
+      {
+        reply_ = network_manager_->head (request);
+      }
+    else
+      {
+        reply_ = network_manager_->get (request);
+      }
+
+    connect (reply_.data (), &QNetworkReply::finished, this, &LotWUsers::impl::reply_finished);
+    connect (reply_.data (), &QNetworkReply::readyRead, this, &LotWUsers::impl::store);
+  }
+
+  void reply_finished ()
+  {
+    if (!reply_) return;        // we probably deleted it in an
+                                // earlier call
+    QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()};
+    if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ())
+      {
+        if (++redirect_count_ < 10) // maintain sanity
+          {
+            // follow redirect
+            download (reply_->url ().resolved (redirect_url));
+          }
+        else
+          {
+            Q_EMIT self_->LotW_users_error (tr ("Network Error - Too many redirects:\n\'%1\'")
+                                            .arg (redirect_url.toDisplayString ()));
+            url_valid_ = false; // reset
+          }
+      }
+    else if (reply_->error () != QNetworkReply::NoError)
+      {
+        csv_file_.cancelWriting ();
+        csv_file_.commit ();
+        url_valid_ = false;     // reset
+        // report errors that are not due to abort
+        if (QNetworkReply::OperationCanceledError != reply_->error ())
+          {
+            Q_EMIT self_->LotW_users_error (tr ("Network Error:\n%1")
+                                            .arg (reply_->errorString ()));
+          }
+      }
+    else
+      {
+        if (url_valid_ && !csv_file_.commit ())
+          {
+            Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot commit changes to:\n\"%1\"")
+                                            .arg (csv_file_.fileName ()));
+            url_valid_ = false; // reset
+          }
+        else
+          {
+            if (!url_valid_)
+              {
+                // now get the body content
+                url_valid_ = true;
+                download (reply_->url ().resolved (redirect_url));
+              }
+            else
+              {
+                url_valid_ = false; // reset
+                qDebug () << "LotW Users Data downloaded from" << reply_->url ().toDisplayString ();
+                // load the database asynchronously
+                future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_.fileName ());
+              }
+          }
+      }
+    if (reply_ && reply_->isFinished ())
+      {
+        reply_->deleteLater ();
+      }
+  }
+
+  void store ()
+  {
+    if (url_valid_)
+      {
+        if (!csv_file_.isOpen ())
+          {
+            // create temporary file in the final location
+            if (!csv_file_.open (QSaveFile::WriteOnly))
+              {
+                abort ();
+                Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot open file:\n\"%1\"\nError(%2): %3")
+                                                .arg (csv_file_.fileName ())
+                                                .arg (csv_file_.error ())
+                                                .arg (csv_file_.errorString ()));
+              }
+          }
+        if (csv_file_.write (reply_->read (reply_->bytesAvailable ())) < 0)
+          {
+            abort ();
+            Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot write to file:\n\"%1\"\nError(%2): %3")
+                                            .arg (csv_file_.fileName ())
+                                            .arg (csv_file_.error ())
+                                            .arg (csv_file_.errorString ()));
+          }
+      }
+  }
+
+  void abort ()
+  {
+    if (reply_ && reply_->isRunning ())
+      {
+        reply_->abort ();
+      }
+  }
 
   // Load the database from the given file name
   //
   // Expects the file to be in CSV format with no header with one
   // record per line. Record fields are call sign followed by upload
   // date in yyyy-MM-dd format followed by upload time (ignored)
-  dictionary load (QString const& lotw_users_file)
+  dictionary load_dictionary (QString const& lotw_csv_file)
   {
     dictionary result;
-    QFile f {lotw_users_file};
+    QFile f {lotw_csv_file};
     if (f.open (QFile::ReadOnly | QFile::Text))
       {
         QTextStream s {&f};
@@ -46,26 +212,39 @@ namespace
       }
     return result;
   }
-}
 
-class LotWUsers::impl final
+  LotWUsers * self_;
-{
+  QNetworkAccessManager * network_manager_;
-public:
+  QSaveFile csv_file_;
+  bool url_valid_;
+  QUrl current_url_;            // may be a redirect
+  int redirect_count_;
+  QPointer<QNetworkReply> reply_;
   std::future<dictionary> future_load_;
   dictionary last_uploaded_;
 };
 
-LotWUsers::LotWUsers (Configuration const * configuration, QObject * parent)
+#include "LotWUsers.moc"
+
+LotWUsers::LotWUsers (Configuration const * configuration, QNetworkAccessManager * network_manager
+                      , QObject * parent)
   : QObject {parent}
+  , m_ {this
+        , network_manager
+        , configuration->writeable_data_dir ().absoluteFilePath ("lotw-user-activity.csv")}
 {
-  // load the database asynchronously
+  m_->load (false);
-  m_->future_load_ = std::async (std::launch::async, load, configuration->writeable_data_dir ().absoluteFilePath ("lotw-user-activity.csv"));
 }
 
 LotWUsers::~LotWUsers ()
 {
 }
 
+void LotWUsers::download_new_file ()
+{
+  m_->load (true);
+}
+
 bool LotWUsers::user (QString const& call, qint64 uploaded_since_days) const
 {
   if (m_->future_load_.valid ())

LotWUsers.hpp

@@ -8,6 +8,7 @@
 class QString;
 class QDate;
 class Configuration;
+class QNetworkAccessManager;
 
 //
 // LotWUsers - Lookup Logbook of the World users
@@ -18,9 +19,11 @@ class LotWUsers final
   Q_OBJECT
 
 public:
-  LotWUsers (Configuration const * configuration, QObject * parent = 0);
+  LotWUsers (Configuration const * configuration, QNetworkAccessManager *, QObject * parent = 0);
   ~LotWUsers ();
 
+  void download_new_file ();
+
   // returns true if the specified call sign 'call' has uploaded their
   // log to LotW in the last 'uploaded_since_days' days
   Q_SLOT bool user (QString const& call, qint64 uploaded_since_days) const;

SampleDownloader/RemoteFile.cpp

@@ -33,6 +33,8 @@ void RemoteFile::local_file_path (QString const& name)
         {
           QFile file {local_file_.absoluteFilePath ()};
           if (!file.rename (new_file.absoluteFilePath ()))
+            {
+              if (listener_)
                 {
                   listener_->error (tr ("File System Error")
                                     , tr ("Cannot rename file:\n\"%1\"\nto: \"%2\"\nError(%3): %4")
@@ -42,6 +44,7 @@ void RemoteFile::local_file_path (QString const& name)
                                     .arg (file.errorString ()));
                 }
             }
+        }
       std::swap (local_file_, new_file);
     }
 }
@@ -49,6 +52,8 @@ void RemoteFile::local_file_path (QString const& name)
 bool RemoteFile::local () const
 {
   auto is_local = (reply_ && !reply_->isFinished ()) || local_file_.exists ();
+  if (listener_)
+    {
       if (is_local)
         {
           auto size = local_file_.size ();
@@ -59,6 +64,7 @@ bool RemoteFile::local () const
         {
           listener_->download_progress (-1, 0);
         }
+    }
   return is_local;
 }
 
@@ -91,14 +97,20 @@ bool RemoteFile::sync (QUrl const& url, bool local, bool force)
         {
           auto path = local_file_.absoluteDir ();
           if (path.remove (local_file_.fileName ()))
+            {
+              if (listener_)
                 {
                   listener_->download_progress (-1, 0);
                 }
+            }
           else
+            {
+              if (listener_)
                 {
                   listener_->error (tr ("File System Error")
                                     , tr ("Cannot delete file:\n\"%1\"")
                                     .arg (local_file_.absoluteFilePath ()));
+                }
               return false;
             }
           path.rmpath (".");
@@ -137,11 +149,14 @@ void RemoteFile::download (QUrl url)
   connect (reply_.data (), &QNetworkReply::readyRead, this, &RemoteFile::store);
   connect (reply_.data (), &QNetworkReply::downloadProgress
            , [this] (qint64 bytes_received, qint64 total_bytes) {
+             if (listener_)
+               {
                  // report progress of wanted file
                  if (is_valid_)
                    {
                      listener_->download_progress (bytes_received, total_bytes);
                    }
+               }
            });
 }
 
@@ -160,7 +175,7 @@ void RemoteFile::reply_finished ()
   QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()};
   if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ())
     {
-      if (listener_->redirect_request (redirect_url))
+      if (!listener_ || listener_->redirect_request (redirect_url))
         {
           if (++redirect_count_ < 10) // maintain sanity
             {
@@ -168,20 +183,26 @@ void RemoteFile::reply_finished ()
               download (reply_->url ().resolved (redirect_url));
             }
           else
+            {
+              if (listener_)
                 {
                   listener_->download_finished (false);
                   listener_->error (tr ("Network Error")
                                     , tr ("Too many redirects: %1")
                                     .arg (redirect_url.toDisplayString ()));
+                }
               is_valid_ = false; // reset
             }
         }
       else
+        {
+          if (listener_)
             {
               listener_->download_finished (false);
               listener_->error (tr ("Network Error")
                                 , tr ("Redirect not followed: %1")
                                 .arg (redirect_url.toDisplayString ()));
+            }
           is_valid_ = false;    // reset
         }
     }
@@ -189,10 +210,13 @@ void RemoteFile::reply_finished ()
     {
       file_.cancelWriting ();
       file_.commit ();
+      if (listener_)
+        {
           listener_->download_finished (false);
+        }
       is_valid_ = false;        // reset
       // report errors that are not due to abort
-      if (QNetworkReply::OperationCanceledError != reply_->error ())
+      if (listener_ && QNetworkReply::OperationCanceledError != reply_->error ())
         {
           listener_->error (tr ("Network Error"), reply_->errorString ());
         }
@@ -201,12 +225,18 @@ void RemoteFile::reply_finished ()
     {
       auto path = QFileInfo {file_.fileName ()}.absoluteDir ();
       if (is_valid_ && !file_.commit ())
+        {
+          if (listener_)
             {
               listener_->error (tr ("File System Error")
                                 , tr ("Cannot commit changes to:\n\"%1\"")
                                 .arg (file_.fileName ()));
+            }
           path.rmpath (".");    // tidy empty directories
+          if (listener_)
+            {
               listener_->download_finished (false);
+            }
           is_valid_ = false;    // reset
         }
       else
@@ -218,8 +248,11 @@ void RemoteFile::reply_finished ()
               download (reply_->url ().resolved (redirect_url));
             }
           else
+            {
+              if (listener_)
                 {
                   listener_->download_finished (true);
+                }
               is_valid_ = false; // reset
             }
         }
@@ -243,6 +276,8 @@ void RemoteFile::store ()
               if (!file_.open (QSaveFile::WriteOnly))
                 {
                   abort ();
+                  if (listener_)
+                    {
                       listener_->error (tr ("File System Error")
                                         , tr ("Cannot open file:\n\"%1\"\nError(%2): %3")
                                         .arg (path.path ())
@@ -250,17 +285,23 @@ void RemoteFile::store ()
                                         .arg (file_.errorString ()));
                     }
                 }
+            }
           else
             {
               abort ();
+              if (listener_)
+                {
                   listener_->error (tr ("File System Error")
                                     , tr ("Cannot make path:\n\"%1\"")
                                     .arg (path.path ()));
                 }
             }
+        }
       if (file_.write (reply_->read (reply_->bytesAvailable ())) < 0)
         {
           abort ();
+          if (listener_)
+            {
               listener_->error (tr ("File System Error")
                                 , tr ("Cannot write to file:\n\"%1\"\nError(%2): %3")
                                 .arg (file_.fileName ())
@@ -269,3 +310,4 @@ void RemoteFile::store ()
             }
         }
     }
+}

mainwindow.cpp

@@ -203,7 +203,7 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple,
   m_settings {multi_settings->settings ()},
   ui(new Ui::MainWindow),
   m_config {temp_directory, m_settings, this},
-  m_lotw_users {&m_config},
+  m_lotw_users {&m_config, &m_network_manager},
   m_WSPR_band_hopping {m_settings, &m_config, this},
   m_WSPR_tx_next {false},
   m_rigErrorMessageBox {MessageBox::Critical, tr ("Rig Control Error")