FT8 demod: OSD optimization and fixes

ft8/fftplan.h

@@ -0,0 +1,62 @@
+///////////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2023 Edouard Griffiths, F4EXB.                                  //
+//                                                                               //
+// This is the code from ft8mon: https://github.com/rtmrtmrtmrtm/ft8mon          //
+// written by Robert Morris, AB1HL                                               //
+// reformatted and adapted to Qt and SDRangel context                            //
+//                                                                               //
+// This program is free software; you can redistribute it and/or modify          //
+// it under the terms of the GNU General Public License as published by          //
+// the Free Software Foundation as version 3 of the License, or                  //
+// (at your option) any later version.                                           //
+//                                                                               //
+// This program is distributed in the hope that it will be useful,               //
+// but WITHOUT ANY WARRANTY; without even the implied warranty of                //
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                  //
+// GNU General Public License V3 for more details.                               //
+//                                                                               //
+// You should have received a copy of the GNU General Public License             //
+// along with this program. If not, see <http://www.gnu.org/licenses/>.          //
+///////////////////////////////////////////////////////////////////////////////////
+
+#ifndef FFTPLAN_H
+#define FFTPLAN_H
+
+#include <fftw3.h>
+
+namespace FT8
+{
+
+    // a cached fftw plan, for both of:
+    // fftwf_plan_dft_r2c_1d(n, m_in, m_out, FFTW_ESTIMATE);
+    // fftwf_plan_dft_c2r_1d(n, m_in, m_out, FFTW_ESTIMATE);
+    class Plan
+    {
+    public:
+        int n_;
+        int type_;
+
+        //
+        // real -> complex
+        //
+        fftwf_complex *c_; // (n_ / 2) + 1 of these
+        float *r_;         // n_ of these
+        fftwf_plan fwd_;   // forward plan
+        fftwf_plan rev_;   // reverse plan
+
+        //
+        // complex -> complex
+        //
+        fftwf_complex *cc1_; // n
+        fftwf_complex *cc2_; // n
+        fftwf_plan cfwd_;    // forward plan
+        fftwf_plan crev_;    // reverse plan
+
+        static const int M_FFTW_TYPE = FFTW_ESTIMATE;
+    }; // Plan
+
+    // MEASURE=0, ESTIMATE=64, PATIENT=32
+
+} // namespace FT8
+
+#endif // FFTPLAN_H

ft8/unpack.cpp

@@ -289,15 +289,15 @@ std::string Packing::unpack_4(int a77[], std::string& call1str, std::string& cal
 
     if (swap)
     {
-        msg = std::string(call) + " " + ocall;
+        msg = call1str + " " + ocall;
-        call1str = call;
+        call1str = trim(call);
-        call2str = ocall;
+        call2str = trim(ocall);
     }
     else
     {
         msg = std::string(ocall) + " " + call;
-        call1str = ocall;
+        call1str = trim(ocall);
-        call2str = call;
+        call2str = trim(call);
     }
 
     int suffix = un64(a77, 71, 2);

plugins/channelrx/demodft8/ft8demodworker.cpp

@@ -18,7 +18,6 @@
 #include <QStandardPaths>
 #include <QDir>
 #include <QDateTime>
-#include <QMutableListIterator>
 
 #include "channel/channelapi.h"
 #include "dsp/wavfilerecord.h"
@@ -73,8 +72,14 @@ int FT8DemodWorker::FT8Callback::hcb(
     }
 
     cycle_already[msg] = true;
+    QString call2Str(call2.c_str());
     QString info(comment);
-    QString call2QStr(call2.c_str());
+
+    if (m_validCallsigns && info.startsWith("OSD") && !m_validCallsigns->contains(call2Str))
+    {
+        cycle_mu.unlock();
+        return 2; // leave without reporting. Set as new decode.
+    }
 
     QList<FT8Message>& ft8Messages = m_msgReportFT8Messages->getFT8Messages();
     FT8Message baseMessage{
@@ -86,7 +91,7 @@ int FT8DemodWorker::FT8Callback::hcb(
         off - 0.5f,
         hz0,
         QString(call1.c_str()).simplified(),
-        call2QStr,
+        call2Str,
         QString(loc.c_str()).simplified(),
         info
     };
@@ -221,20 +226,6 @@ void FT8DemodWorker::processBuffer(int16_t *buffer, QDateTime periodTS)
     qDebug("FT8DemodWorker::processBuffer: done: at %6.3f %d messages",
         m_baseFrequency / 1000000.0, ft8Callback.getReportMessage()->getFT8Messages().size());
 
-    if (m_useOSD && m_verifyOSD)
-    {
-        QMutableListIterator<FT8Message> i(ft8Callback.getReportMessage()->getFT8Messages());
-
-        while (i.hasNext())
-        {
-            const auto& ft8Message = i.next();
-
-            if (ft8Message.decoderInfo.startsWith("OSD") && !m_validCallsigns.contains(ft8Message.call2)) {
-                i.remove();
-            }
-        }
-    }
-
     if (m_reportingMessageQueue) {
         m_reportingMessageQueue->push(new MsgReportFT8Messages(*ft8Callback.getReportMessage()));
     }
@@ -266,7 +257,7 @@ void FT8DemodWorker::processBuffer(int16_t *buffer, QDateTime periodTS)
                 continue;
             }
 
-            QString logMessage = QString("%1 %2 Rx FT8 %3 %4 %5 %6 %7 %8 %9")
+            QString logMessage = QString("%1 %2 Rx FT8 %3 %4 %5 %6 %7 %8")
                 .arg(periodTS.toString("yyyyMMdd_HHmmss"))
                 .arg(baseFrequencyMHz, 9, 'f', 3)
                 .arg(ft8Message.snr, 6)
@@ -274,8 +265,7 @@ void FT8DemodWorker::processBuffer(int16_t *buffer, QDateTime periodTS)
                 .arg(ft8Message.df, 4, 'f', 0)
                 .arg(ft8Message.call1)
                 .arg(ft8Message.call2)
-                .arg(ft8Message.loc)
+                .arg(ft8Message.loc);
-                .arg(ft8Message.decoderInfo);
             logMessage.remove(0, 2);
             logFile << logMessage.toStdString() << std::endl;
         }

plugins/channelrx/demodft8/readme.md

@@ -140,13 +140,13 @@ Empties the message table (C.10)
 
 <h3>C.8: Log messages</h3>
 
-Toggles the logging of messages. Messages will be logged in the same format as the original WSJT-X format except if the message has decoder information in which case this information is appended at the end of the line.
+Toggles the logging of messages. Messages will be logged in the same format as the original WSJT-X format.
 
 Example:
 
 <code><pre>
-230128_003030    10.136 Rx FT8    -22  0.1 2049 KE0DKZ W2ZI EL99 OSD-0-71
+230128_003030    10.136 Rx FT8    -22  0.1 2049 KE0DKZ W2ZI EL99
----- 1 ------    --2---           -3-  -4- --5- --6--- --7- --8- ---9----
+---- 1 ------    --2---           -3-  -4- --5- --6--- --7- --8-
 </pre></code>
 
   - **1**: Date and time of decoder slot in YYMMDD_HHmmss fomat
@@ -157,7 +157,6 @@ Example:
   - **6**: First callsign area. May contain spaces (ex: "CQ DX"). Note that for messages types 0.1 and 5 it is slighlty different from the standard (See C.10)
   - **7**: Second callsign area and is always a callsign. This might be slighlty different from the standard (see above)
   - **8**: Locator area maybe a 4 or 6 digit locator, a report, acknowledgement (RRR) or a greetings (RR73, 73)
-  - **9**: Decoder information if any
 
 The splitting and naming of files is different from WSJT-X scheme.
 
@@ -220,9 +219,7 @@ This is the time in seconds after which the decoder threads will be prompted to
 
 <h4>C.1.3: Toggle Ordered Statistics Decoding</h4>
 
-This toggles the Ordered Statistics Decoding (OSD). OSD is used if the CRC check fails after LDPC processing. Be careful with OSD as it will try to find solutions that validate the CRC but these solutions can be wrong and thus yield false messages. When post processing the results it is recommended to check against a database of valid callsigns. At least you should use a list of valid country prefixes and test the locator against the country prefix.
+This toggles the Ordered Statistics Decoding (OSD). OSD is used if the CRC check fails after LDPC processing. Be careful with OSD as it will try to find solutions that validate the CRC but these solutions can be wrong and thus yield false messages. To mitigate this you may engage the callsign verification function (C.1.6). Ultimately you may post process the results (logs) through a list of valid country prefixes and correlate country and locator.
-
-With reasonable depth (C.1.4) and minimum correct LDPC bits (C.1.5) values the amount of false messages should be low so OSD may still be interesting. However if you require best accuracy you should filter messages in a post processing step as suggested above.
 
 <h4>C.1.4: Ordered Statistics Decoding depth</h4>
 
@@ -232,7 +229,11 @@ Sets the maximum depth of OSD search.
 
 Sets the minimum number of correct LDPC bits (out of 83) necessary to trigger OSD.
 
-<h4>C.1.6: Band presets table</h4>
+<h4>C.1.6. Verify callsigns</h4>
+
+OSD search may find invalid solutions as mentioned above. When checking this option the callsigns in messages not passed through OSD (thus very certainly valid) are stored for the life of the plugin. When OSD is engaged the second callsign field which is always a callsign is checked against this list and if the callsign is not found the message is rejected. This is quite efficient in removing false messages when OSD is engaged although some valid messages may be removed.
+
+<h4>C.1.7: Band presets table</h4>
 
 This table shows the band presets values that will appear in (C.5)
 
@@ -242,23 +243,23 @@ This table shows the band presets values that will appear in (C.5)
 
 You can edit these values by clicking on the cell in the table.
 
-<h4>C.1.7: Add preset</h4>
+<h4>C.1.8: Add preset</h4>
 
 Use this button to create a new preset. It will take the values from the row of the selected cell in the table (if selected) and put the new preset at the bottom of the table
 
-<h4>C.1.8: Delete preset</h4>
+<h4>C.1.9: Delete preset</h4>
 
 Delete the preset designated by the selected cell in the table.
 
-<h4>C.1.9: Move up preset</h4>
+<h4>C.1.10: Move up preset</h4>
 
 Move up the preset designated by the selected cell in the table.
 
-<h4>C.1.10: Move down preset</h4>
+<h4>C.1.11: Move down preset</h4>
 
 Move down the preset designated by the selected cell in the table.
 
-<h4>C.1.11: Restore defaults</h4>
+<h4>C.1.12: Restore defaults</h4>
 
 This restores the default band preset values:
 
@@ -280,10 +281,10 @@ This restores the default band preset values:
 
 Channel offsets are all set to 0 kHz.
 
-<h4>C.1.12 Commit changes</h4>
+<h4>C.1.13 Commit changes</h4>
 
 Click on the "OK" button to commit changes and close dialog.
 
-<h4>C.1.13 Cancel changes</h4>
+<h4>C.1.14 Cancel changes</h4>
 
 Click on the "Cancel" button to close dialog without making changes.