Scan ADIF log asynchronously and new settings button to rescan ADIF log

CMakeLists.txt

@@ -271,6 +271,8 @@ set (wsjt_qt_CXXSRCS
   item_delegates/CallsignDelegate.cpp
   item_delegates/MaidenheadLocatorDelegate.cpp
   models/CabrilloLog.cpp
+  logbook/AD1CCty.cpp
+  logbook/WorkedBefore.cpp
   )
 
 set (wsjt_qtmm_CXXSRCS
@@ -288,8 +290,6 @@ set (jt9_CXXSRCS
 
 set (wsjtx_CXXSRCS
   logbook/logbook.cpp
-  logbook/WorkedBefore.cpp
-  logbook/AD1CCty.cpp
   psk_reporter.cpp
   Modulator.cpp
   Detector.cpp

Configuration.cpp

@@ -182,6 +182,7 @@
 #include "validators/CallsignValidator.hpp"
 #include "LotWUsers.hpp"
 #include "models/DecodeHighlightingModel.hpp"
+#include "logbook/logbook.h"
 
 #include "ui_Configuration.h"
 #include "moc_Configuration.cpp"
@@ -395,6 +396,7 @@ public:
                  , QNetworkAccessManager * network_manager
                  , QDir const& temp_directory
                  , QSettings * settings
+                 , LogBook * logbook
                  , QWidget * parent);
   ~impl ();
 
@@ -480,6 +482,7 @@ private:
   Q_SLOT void handle_transceiver_update (TransceiverState const&, unsigned sequence_number);
   Q_SLOT void handle_transceiver_failure (QString const& reason);
   Q_SLOT void on_reset_highlighting_to_defaults_push_button_clicked (bool);
+  Q_SLOT void on_rescan_log_push_button_clicked (bool);
   Q_SLOT void on_LotW_CSV_fetch_push_button_clicked (bool);
   Q_SLOT void on_cbx2ToneSpacing_clicked(bool);
   Q_SLOT void on_cbx4ToneSpacing_clicked(bool);
@@ -502,6 +505,7 @@ private:
 
   QNetworkAccessManager * network_manager_;
   QSettings * settings_;
+  LogBook * logbook_;
 
   QDir doc_dir_;
   QDir data_dir_;
@@ -638,8 +642,8 @@ private:
 
 // delegate to implementation class
 Configuration::Configuration (QNetworkAccessManager * network_manager, QDir const& temp_directory,
-                              QSettings * settings, QWidget * parent)
+                              QSettings * settings, LogBook * logbook, QWidget * parent)
-  : m_ {this, network_manager, temp_directory, settings, parent}
+  : m_ {this, network_manager, temp_directory, settings, logbook, parent}
 {
 }
 
@@ -905,13 +909,15 @@ namespace
 }
 
 Configuration::impl::impl (Configuration * self, QNetworkAccessManager * network_manager
-                           , QDir const& temp_directory, QSettings * settings, QWidget * parent)
+                           , QDir const& temp_directory, QSettings * settings, LogBook * logbook
+                           , QWidget * parent)
   : QDialog {parent}
   , self_ {self}
   , transceiver_thread_ {nullptr}
   , ui_ {new Ui::configuration_dialog}
   , network_manager_ {network_manager}
   , settings_ {settings}
+  , logbook_ {logbook}
   , doc_dir_ {doc_path ()}
   , data_dir_ {data_path ()}
   , temp_dir_ {temp_directory}
@@ -2126,6 +2132,11 @@ void Configuration::impl::on_reset_highlighting_to_defaults_push_button_clicked
     }
 }
 
+void Configuration::impl::on_rescan_log_push_button_clicked (bool /*clicked*/)
+{
+  if (logbook_) logbook_->rescan ();
+}
+
 void Configuration::impl::on_LotW_CSV_fetch_push_button_clicked (bool /*checked*/)
 {
   lotw_users_.load (ui_->LotW_CSV_URL_line_edit->text (), true);

Configuration.hpp

@@ -24,6 +24,7 @@ class QStringListModel;
 class QHostAddress;
 class LotWUsers;
 class DecodeHighlightingModel;
+class LogBook;
 
 //
 // Class Configuration
@@ -72,7 +73,7 @@ public:
   Q_ENUM (Type2MsgGen)
 
   explicit Configuration (QNetworkAccessManager *, QDir const& temp_directory, QSettings * settings,
-                          QWidget * parent = nullptr);
+                          LogBook * logbook, QWidget * parent = nullptr);
   ~Configuration ();
 
   void select_tab (int);

Configuration.ui

@@ -2233,8 +2233,8 @@ Right click for insert and delete options.</string>
            </widget>
           </item>
           <item>
-           <layout class="QGridLayout" name="gridLayout_5">
+           <layout class="QHBoxLayout" name="horizontalLayout_15">
-            <item row="0" column="0">
+            <item>
              <widget class="QCheckBox" name="highlight_by_mode_check_box">
               <property name="toolTip">
                <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Check to indicate new DXCC entities, grid squares, and callsigns per mode.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
@@ -2244,6 +2244,26 @@ Right click for insert and delete options.</string>
               </property>
              </widget>
             </item>
+            <item>
+             <spacer name="horizontalSpacer_5">
+              <property name="orientation">
+               <enum>Qt::Horizontal</enum>
+              </property>
+              <property name="sizeHint" stdset="0">
+               <size>
+                <width>40</width>
+                <height>20</height>
+               </size>
+              </property>
+             </spacer>
+            </item>
+            <item>
+             <widget class="QPushButton" name="rescan_log_push_button">
+              <property name="text">
+               <string>Rescan ADIF Log</string>
+              </property>
+             </widget>
+            </item>
            </layout>
           </item>
          </layout>
@@ -2924,7 +2944,6 @@ Right click for insert and delete options.</string>
   <tabstop>stations_table_view</tabstop>
   <tabstop>highlighting_list_view</tabstop>
   <tabstop>reset_highlighting_to_defaults_push_button</tabstop>
-  <tabstop>highlight_by_mode_check_box</tabstop>
   <tabstop>LotW_CSV_URL_line_edit</tabstop>
   <tabstop>LotW_CSV_fetch_push_button</tabstop>
   <tabstop>LotW_days_since_upload_spin_box</tabstop>

logbook/WorkedBefore.cpp

@@ -1,22 +1,29 @@
 #include "WorkedBefore.hpp"
 
 #include <functional>
+#include <stdexcept>
 #include <boost/functional/hash.hpp>
 #include <boost/multi_index_container.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
 #include <boost/multi_index/key_extractors.hpp>
+#include <QtConcurrent/QtConcurrentRun>
+#include <QFuture>
+#include <QFutureWatcher>
 #include <QChar>
 #include <QString>
 #include <QByteArray>
 #include <QStandardPaths>
 #include <QDir>
+#include <QFileInfo>
 #include <QFile>
 #include <QTextStream>
-#include <QDebug>
+
 #include "qt_helpers.hpp"
 #include "pimpl_impl.hpp"
 
+#include "moc_WorkedBefore.cpp"
+
 using namespace boost::multi_index;
 
 // hash function for QString members in hashed indexes
@@ -214,6 +221,20 @@ namespace
 {
   auto const logFileName = "wsjtx_log.adi";
 
+  // Expception class suitable for using with QtConcurrent across
+  // thread boundaries
+  class LoaderException final
+    : public QException
+  {
+  public:
+    LoaderException (std::exception const& e) : error_ {e.what ()} {}
+    QString error () const {return error_;}
+    void raise () const override {throw *this;}
+    LoaderException * clone () const override {return new LoaderException {*this};}
+  private:
+    QString error_;
+  };
+
   QString extractField (QString const& record, QString const& fieldName)
   {
     int fieldNameIndex = record.indexOf ('<' + fieldName + ':', 0, Qt::CaseInsensitive);
@@ -228,8 +249,12 @@ namespace
             if (dataTypeIndex > closingBracketIndex)
               dataTypeIndex = -1; // second : was found but it was beyond the closing >
           }
+        else
+          {
+            throw LoaderException (std::runtime_error {"Invalid ADIF field " + fieldName.toStdString () + ": " + record.toStdString ()});
+          }
 
-        if ((closingBracketIndex > fieldNameIndex) && (fieldLengthIndex > fieldNameIndex) && (fieldLengthIndex< closingBracketIndex))
+        if (closingBracketIndex > fieldNameIndex && fieldLengthIndex > fieldNameIndex && fieldLengthIndex < closingBracketIndex)
           {
             int fieldLengthCharCount = closingBracketIndex - fieldLengthIndex -1;
             if (dataTypeIndex >= 0)
@@ -241,8 +266,93 @@ namespace
                 return record.mid(closingBracketIndex+1,fieldLength);
               }
           }
+        else
+          {
+            throw LoaderException (std::runtime_error {"Malformed ADIF field " + fieldName.toStdString () + ": " + record.toStdString ()});
+          }
       }
-    return "";
+    return QString {};
+  }
+
+  worked_before_database_type loader (QString const& path, AD1CCty const * prefixes)
+  {
+    worked_before_database_type worked;
+    QFile inputFile {path};
+    if (inputFile.exists ())
+      {
+        if (inputFile.open (QFile::ReadOnly))
+          {
+            QTextStream in {&inputFile};
+            QString buffer;
+            bool pre_read {false};
+            int end_position {-1};
+
+            // skip optional header record
+            do
+              {
+                buffer += in.readLine () + '\n';
+                if (buffer.startsWith (QChar {'<'})) // denotes no header
+                  {
+                    pre_read = true;
+                  }
+                else
+                  {
+                    end_position = buffer.indexOf ("<EOH>", 0, Qt::CaseInsensitive);
+                  }
+              }
+            while (!in.atEnd () && !pre_read && end_position < 0);
+            if (!pre_read)            // found header
+              {
+                if (end_position < 0)
+                  {
+                    throw LoaderException (std::runtime_error {"Invalid ADIF header"});
+                  }
+                buffer.remove (0, end_position + 5);
+              }
+            while (!in.atEnd ())
+              {
+                end_position = buffer.indexOf ("<EOR>", 0, Qt::CaseInsensitive);
+                do
+                  {
+                    if (!in.atEnd () && end_position < 0)
+                      {
+                        buffer += in.readLine () + '\n';
+                      }
+                  }
+                while ((end_position = buffer.indexOf ("<EOR>", 0, Qt::CaseInsensitive)) < 0 && !in.atEnd ());
+                if (end_position < 0)
+                  {
+                    throw LoaderException (std::runtime_error {"Invalid ADIF record starting at: " + buffer.left (40).toStdString ()});
+                  }
+                auto record = buffer.left (end_position + 5).trimmed ();
+                auto next_record = buffer.indexOf (QChar {'<'}, end_position + 5);
+                buffer.remove (0, next_record >=0 ? next_record : buffer.size ());
+                record = record.mid (record.indexOf (QChar {'<'}));
+                auto call = extractField (record, "CALL");
+                if (call.size ())
+                  {
+                    auto const& entity = prefixes->lookup (call);
+                    worked.emplace (call.toUpper ()
+                                    , extractField (record, "GRIDSQUARE").left (4).toUpper () // not interested in 6-digit grids
+                                    , extractField (record, "BAND").toUpper ()
+                                    , extractField (record, "MODE").toUpper ()
+                                    , entity.entity_name
+                                    , entity.continent
+                                    , entity.CQ_zone
+                                    , entity.ITU_zone);
+                  }
+                else
+                  {
+                    throw LoaderException (std::runtime_error {"Invalid ADIF record with no CALL: " + record.toStdString ()});
+                  }
+              }
+          }
+        else
+          {
+            throw LoaderException (std::runtime_error {"Error opening ADIF log file for read: " + inputFile.errorString ().toStdString ()});
+          }
+      }
+    return worked;
   }
 }
 
@@ -254,70 +364,41 @@ public:
   {
   }
 
+  void reload ()
+  {
+    async_loader_ = QtConcurrent::run (loader, path_, &prefixes_);
+    loader_watcher_.setFuture (async_loader_);
+  }
+
   QString path_;
   AD1CCty prefixes_;
+  QFutureWatcher<worked_before_database_type> loader_watcher_;
+  QFuture<worked_before_database_type> async_loader_;
   worked_before_database_type worked_;
 };
 
 WorkedBefore::WorkedBefore ()
 {
-  QFile inputFile {m_->path_};
+  connect (&m_->loader_watcher_, QFutureWatcher<worked_before_database_type>::finished, [this] () {
-  if (inputFile.open (QFile::ReadOnly))
+      QString error;
-    {
+      size_t n {0};
-      QTextStream in {&inputFile};
+      try
-      QString buffer;
+        {
-      bool pre_read {false};
+          m_->worked_ = m_->loader_watcher_.result ();
-      int end_position {-1};
+          n = m_->worked_.size ();
+        }
+      catch (LoaderException const& e)
+        {
+          error = e.error ();
+        }
+      Q_EMIT finished_loading (n, error);
+    });
+  reload ();
+}
 
-      // skip optional header record
+void WorkedBefore::reload ()
-      do
+{
-        {
+  m_->reload ();
-          buffer += in.readLine () + '\n';
-          if (buffer.startsWith (QChar {'<'})) // denotes no header
-            {
-              pre_read = true;
-            }
-          else
-            {
-              end_position = buffer.indexOf ("<EOH>", 0, Qt::CaseInsensitive);
-            }
-        }
-      while (!in.atEnd () && !pre_read && end_position < 0);
-      if (!pre_read)            // found header
-        {
-          buffer.remove (0, end_position + 5);
-        }
-      while (buffer.size () || !in.atEnd ())
-        {
-          do
-            {
-              end_position = buffer.indexOf ("<EOR>", 0, Qt::CaseInsensitive);
-              if (!in.atEnd () && end_position < 0)
-                {
-                  buffer += in.readLine () + '\n';
-                }
-            }
-          while (!in.atEnd () && end_position < 0);
-          int record_length {end_position >= 0 ? end_position + 5 : -1};
-          auto record = buffer.left (record_length).trimmed ();
-          auto next_record = buffer.indexOf (QChar {'<'}, record_length);
-          buffer.remove (0, next_record >=0 ? next_record : buffer.size ());
-          record = record.mid (record.indexOf (QChar {'<'}));
-          auto call = extractField (record, "CALL");
-          if (call.size ())
-            {
-              auto const& entity = m_->prefixes_.lookup (call);
-              m_->worked_.emplace (call.toUpper ()
-                                   , extractField (record, "GRIDSQUARE").left (4).toUpper () // not interested in 6-digit grids
-                                   , extractField (record, "BAND").toUpper ()
-                                   , extractField (record, "MODE").toUpper ()
-                                   , entity.entity_name
-                                   , entity.continent
-                                   , entity.CQ_zone
-                                   , entity.ITU_zone);
-            }
-        }
-    }
 }
 
 WorkedBefore::~WorkedBefore ()

logbook/WorkedBefore.hpp

@@ -1,7 +1,7 @@
 #ifndef WORKWED_BEFORE_HPP_
 #define WORKWED_BEFORE_HPP_
 
-#include <boost/core/noncopyable.hpp>
+#include <QObject>
 #include "AD1CCty.hpp"
 #include "pimpl_h.hpp"
 
@@ -10,21 +10,25 @@ class QString;
 class QByteArray;
 
 class WorkedBefore final
-  : private boost::noncopyable
+  : public QObject
 {
+  Q_OBJECT
+
 public:
   using Continent = AD1CCty::Continent;
 
   explicit WorkedBefore ();
   ~WorkedBefore ();
 
+  Q_SLOT void reload ();
+  Q_SLOT bool add (QString const& call
+                   , QString const& grid
+                   , QString const& band
+                   , QString const& mode
+                   , QByteArray const& ADIF_record);
+
   QString const& path () const;
   AD1CCty const& countries () const;
-  bool add (QString const& call
-            , QString const& grid
-            , QString const& band
-            , QString const& mode
-            , QByteArray const& ADIF_record);
   bool country_worked (QString const& call, QString const& mode, QString const& band) const;
   bool grid_worked (QString const& grid, QString const& mode, QString const& band) const;
   bool call_worked (QString const& call, QString const& mode, QString const& band) const;
@@ -32,6 +36,8 @@ public:
   bool CQ_zone_worked (int CQ_zone, QString const& mode, QString const& band) const;
   bool ITU_zone_worked (int ITU_zone, QString const& mode, QString const& band) const;
 
+  Q_SIGNAL void finished_loading (int worked_before_record_count, QString const& error) const;
+
 private:
   class impl;
   pimpl<impl> m_;

logbook/logbook.cpp

@@ -1,13 +1,15 @@
 #include "logbook.h"
 
 #include <QDateTime>
-#include <QDebug>
 #include "Configuration.hpp"
 #include "AD1CCty.hpp"
 
+#include "moc_logbook.cpp"
+
 LogBook::LogBook (Configuration const * configuration)
   : config_ {configuration}
 {
+  connect (&worked_before_, &WorkedBefore::finished_loading, this, &LogBook::finished_loading);
 }
 
 void LogBook::match (QString const& call, QString const& mode, QString const& grid,
@@ -52,6 +54,11 @@ bool LogBook::add (QString const& call
   return worked_before_.add (call, grid, band, mode, ADIF_record);
 }
 
+void LogBook::rescan ()
+{
+  worked_before_.reload ();
+}
+
 QByteArray LogBook::QSOToADIF (QString const& hisCall, QString const& hisGrid, QString const& mode,
                                QString const& rptSent, QString const& rptRcvd, QDateTime const& dateTimeOn,
                                QDateTime const& dateTimeOff, QString const& band, QString const& comments,

logbook/logbook.h

@@ -1,12 +1,12 @@
+// -*- Mode: C++ -*-
 /*
  * From an ADIF file and cty.dat, get a call's DXCC entity and its worked before status
- * VK3ACF July 2013
  */
 
 #ifndef LOG_BOOK_H_
 #define LOG_BOOK_H_
 
-#include <boost/core/noncopyable.hpp>
+#include <QObject>
 #include <QString>
 
 #include "WorkedBefore.hpp"
@@ -16,9 +16,11 @@ class QByteArray;
 class QDateTime;
 
 class LogBook final
-  : private boost::noncopyable
+  : public QObject
 {
- public:
+  Q_OBJECT
+
+public:
   LogBook (Configuration const *);
   QString const& path () const {return worked_before_.path ();}
   bool add (QString const& call
@@ -27,6 +29,7 @@ class LogBook final
             , QString const& mode
             , QByteArray const& ADIF_record);
   AD1CCty const& countries () const {return worked_before_.countries ();}
+  void rescan ();
   void match (QString const& call, QString const& mode, QString const& grid,
               AD1CCty::Record const&, bool& callB4, bool& countryB4,
               bool &gridB4, bool &continentB4, bool& CQZoneB4, bool& ITUZoneB4,
@@ -38,7 +41,9 @@ class LogBook final
                                QString const& m_myGrid, QString const& m_txPower, QString const& operator_call,
                                QString const& xSent, QString const& xRcvd);
 
- private:
+  Q_SIGNAL void finished_loading (int worked_before_record_count, QString const& error) const;
+
+private:
   Configuration const * config_;
   WorkedBefore worked_before_;
 };

widgets/mainwindow.cpp

@@ -210,7 +210,8 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple,
   m_configurations_button {0},
   m_settings {multi_settings->settings ()},
   ui(new Ui::MainWindow),
-  m_config {&m_network_manager, temp_directory, m_settings, this},
+  m_logBook {&m_config},
+  m_config {&m_network_manager, temp_directory, m_settings, &m_logBook, this},
   m_WSPR_band_hopping {m_settings, &m_config, this},
   m_WSPR_tx_next {false},
   m_rigErrorMessageBox {MessageBox::Critical, tr ("Rig Control Error")
@@ -361,7 +362,6 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple,
       },
   m_sfx {"P",  "0",  "1",  "2",  "3",  "4",  "5",  "6",  "7",  "8",  "9",  "A"},
   mem_jt9 {shdmem},
-  m_logBook {&m_config},
   m_msAudioOutputBuffered (0u),
   m_framesAudioInputBuffered (RX_SAMPLE_RATE / 10),
   m_downSampleFactor (downSampleFactor),
@@ -463,6 +463,18 @@ MainWindow::MainWindow(QDir const& temp_directory, bool multiple,
   connect (m_logDlg.data (), &LogQSO::acceptQSO, this, &MainWindow::acceptQSO);
   connect (this, &MainWindow::finished, m_logDlg.data (), &LogQSO::close);
 
+  // hook up the log book
+  connect (&m_logBook, &LogBook::finished_loading, [this] (int record_count, QString const& error) {
+      if (error.size ())
+        {
+          MessageBox::warning_message (this, tr ("Error Scanning ADIF Log"), error);
+        }
+      else
+        {
+          showStatusMessage (tr ("Scanned ADIF log, %1 worked before records created").arg (record_count));
+        }
+    });
+
   // Network message handlers
   connect (m_messageClient, &MessageClient::reply, this, &MainWindow::replyToCQ);
   connect (m_messageClient, &MessageClient::replay, this, &MainWindow::replayDecodes);

widgets/mainwindow.h

@@ -347,6 +347,7 @@ private:
   QSettings * m_settings;
   QScopedPointer<Ui::MainWindow> ui;
 
+  LogBook m_logBook;            // must be before Configuration construction
   Configuration m_config;
   WSPRBandHopping m_WSPR_band_hopping;
   bool m_WSPR_tx_next;
@@ -626,7 +627,6 @@ private:
   QDateTime m_dateTimeLastTX;
 
   QSharedMemory *mem_jt9;
-  LogBook m_logBook;
   QString m_QSOText;
   unsigned m_msAudioOutputBuffered;
   unsigned m_framesAudioInputBuffered;