Yet another Thread termination problem found, and try killing the app if the closing turns bad

src/CubicSDR.cpp

@@ -384,39 +384,61 @@ bool CubicSDR::OnInit() {
 int CubicSDR::OnExit() {
 #if USE_HAMLIB
     if (rigIsActive()) {
-        std::cout << "Terminating Rig thread.." << std::endl;
+        std::cout << "Terminating Rig thread.."  << std::endl << std::flush;
         stopRig();
     }
 #endif
 
+    bool terminationSequenceOK = true;
+
     //The thread feeding them all should be terminated first, so: 
-    std::cout << "Terminating SDR thread.." << std::endl;
+    std::cout << "Terminating SDR thread.." << std::endl << std::flush ;
     sdrThread->terminate();
-    sdrThread->isTerminated(3000);
-  
-    std::cout << "Terminating SDR post-processing thread.." << std::endl;
+    terminationSequenceOK = terminationSequenceOK && sdrThread->isTerminated(3000);
+
+    //in case termination sequence goes wrong, kill App brutally now because it can get stuck. 
+    if (!terminationSequenceOK) {
+        //no trace here because it could occur if the device is not started.  
+        ::exit(11);
+    }
+
+    std::cout << "Terminating SDR post-processing thread.." << std::endl << std::flush;
     sdrPostThread->terminate();
 
     //Wait for termination for sdrPostThread second:: since it is doing
     //mostly blocking push() to the other threads, they must stay alive
     //so that sdrPostThread can complete a processing loop and die.
-    sdrPostThread->isTerminated(3000);
+    terminationSequenceOK = terminationSequenceOK && sdrPostThread->isTerminated(3000);
 
-    std::cout << "Terminating All Demodulators.." << std::endl;
+    //in case termination sequence goes wrong, kill App brutally now because it can get stuck. 
+    if (!terminationSequenceOK) {
+        std::cout << "Cannot terminate application properly, calling exit() now." << std::endl << std::flush;
+        ::exit(12);
+    }
+
+    std::cout << "Terminating All Demodulators.." << std::endl << std::flush;
     demodMgr.terminateAll();
+
     //wait for effective death of all demodulators before continuing.
-    demodMgr.garbageCollect(true, 3000);
+    terminationSequenceOK = terminationSequenceOK && demodMgr.garbageCollect(true, 3000);
    
-    std::cout << "Terminating Visual Processor threads.." << std::endl;
+    std::cout << "Terminating Visual Processor threads.." << std::endl << std::flush;
     spectrumVisualThread->terminate();
     if (demodVisualThread) {
         demodVisualThread->terminate();
     }
     
     //Wait nicely
-    spectrumVisualThread->isTerminated(1000);
+    terminationSequenceOK = terminationSequenceOK &&  spectrumVisualThread->isTerminated(1000);
+
     if (demodVisualThread) {
-        demodVisualThread->isTerminated(1000);
+        terminationSequenceOK = terminationSequenceOK && demodVisualThread->isTerminated(1000);
+    }
+
+    //in case termination sequence goes wrong, kill App brutally because it can get stuck. 
+    if (!terminationSequenceOK) {
+        std::cout << "Cannot terminate application properly, calling exit() now." << std::endl << std::flush;
+        ::exit(13);
     }
 
     //Then join the thread themselves:
@@ -469,6 +491,8 @@ int CubicSDR::OnExit() {
     delete m_glContext;
     m_glContext = nullptr;
 
+    std::cout << "Application termination complete." << std::endl << std::flush;
+
 #ifdef __APPLE__
     AudioThread::deviceCleanup();
 #endif

src/demod/DemodulatorMgr.cpp

@@ -278,7 +278,7 @@ DemodulatorInstance *DemodulatorMgr::getLastDemodulatorWith(const std::string& t
 	return nullptr;
 }
 
-void DemodulatorMgr::garbageCollect(bool forcedGC, int maxWaitForTerminationMs) {
+bool DemodulatorMgr::garbageCollect(bool forcedGC, int maxWaitForTerminationMs) {
     
 #define SPIN_WAIT_SLEEP_MS 5
 
@@ -313,7 +313,7 @@ void DemodulatorMgr::garbageCollect(bool forcedGC, int maxWaitForTerminationMs)
 
                 //only garbage collect 1 demod at a time.
                 if (!forcedGC) {
-                    return;
+                    return true;
                 }
             }
             else {
@@ -327,9 +327,11 @@ void DemodulatorMgr::garbageCollect(bool forcedGC, int maxWaitForTerminationMs)
 
         if (currentWaitCycle >= nbCyclesToWait) {
             std::cout << "ERROR: DemodulatorMgr::garbageCollect() has not terminated in time ! (> " << (currentWaitCycle * SPIN_WAIT_SLEEP_MS) << " ms)" << std::endl << std::flush;
-            return;
+            return false;
         }
     } //end while not empty
+
+    return true;
 }
 
 void DemodulatorMgr::updateLastState() {

src/demod/DemodulatorMgr.h

@@ -75,7 +75,8 @@ public:
     //and GC one demod per call. 
     // if forcedGC = true and maxWaitForTerminationMs > 0, do not
     //block the method more than maxWaitForTerminationMs millisecs before returning.
-    void garbageCollect(bool forcedGC = false, int maxWaitForTerminationMs = 0);
+    //Returns: true if forcedGC = false, else true only if all deleted demodulators were GCs before maxWaitForTerminationMs.
+    bool garbageCollect(bool forcedGC = false, int maxWaitForTerminationMs = 0);
     
 private:
 

src/demod/DemodulatorPreThread.cpp

@@ -347,18 +347,22 @@ int DemodulatorPreThread::getAudioSampleRate() {
 }
 
 void DemodulatorPreThread::terminate() {
+
+    //make non-blocking calls to be sure threads are flagged for termination before attempting the blocking calls.
     IOThread::terminate();
+    workerThread->terminate();
+
     DemodulatorThreadIQDataPtr inp(new DemodulatorThreadIQData);    // push dummy to nudge queue
-    
     //VSO: blocking push :
     iqInputQueue->push(inp);
  
-    DemodulatorWorkerThreadCommand command(DemodulatorWorkerThreadCommand::DEMOD_WORKER_THREAD_CMD_NULL);
-    
+    DemodulatorWorkerThreadCommand command;
+    //VSO: blocking push :
     workerQueue->push(command);
     
-    workerThread->terminate();
-    workerThread->isTerminated(1000);
+    //wait blocking for termination here, it could be long with lots of modems and we MUST terminate properly,
+    //else better kill the whole application...
+    workerThread->isTerminated(5000);
 
     t_Worker->join();
     delete t_Worker;

src/demod/DemodulatorThread.cpp

@@ -313,8 +313,8 @@ void DemodulatorThread::run() {
             if (!muted.load() && (!wxGetApp().getSoloMode() || (demodInstance == wxGetApp().getDemodMgr().getLastActiveDemodulator()))) {
                 //non-blocking push needed for audio out
                 if (!audioOutputQueue->try_push(ati)) {
-
-                    std::cout << "DemodulatorThread::run() cannot push ati into audioOutputQueue, is full !" << std::endl;
+                    //Comment this trace, it creates to many false alarms :)
+                    //std::cout << "DemodulatorThread::run() cannot push ati into audioOutputQueue, is full !" << std::endl;
                     std::this_thread::yield();
                 }
             }

src/demod/DemodulatorWorkerThread.cpp

@@ -34,6 +34,7 @@ void DemodulatorWorkerThread::run() {
         //we are waiting for the first command to show up (blocking!)
         //then consuming the commands until done. 
         while (!done && !stopping) {
+
             if (!commandQueue->pop(command, HEARTBEAT_CHECK_PERIOD_MICROS)) {
                 continue;
             }