Improve detection in dead zones (#167)

* Improve detection in dead zones

* Remove elevation check for classification

steno/src/model/classifier/classifier.cpp

@@ -55,8 +55,7 @@ std::vector<std::pair<int, int>> Classifier::getAudioImagePairs(const std::vecto
             // Set the azimuth counter-clockwise to match the audio
             float imagePositionAzimuth = math::getAngleAroundCircle(2.0 * M_PI - imagePositions[j].azimuth - M_PI / 2.f);
 
-            if (std::abs(audioPositions[i].azimuth - imagePositionAzimuth) < rangeThreshold &&
-                std::abs(audioPositions[i].elevation - imagePositions[j].elevation) < rangeThreshold)
+            if (std::abs(audioPositions[i].azimuth - imagePositionAzimuth) < rangeThreshold)
             {
                 audioImagePairs.push_back(std::make_pair(i, j));
             }

steno/src/model/config/config.cpp

@@ -121,7 +121,7 @@ void Config::loadDefault()
     m_dewarpingConfig->setValue(DewarpingConfig::Key::TOP_DISTORSION_FACTOR, 2);
     m_dewarpingConfig->setValue(DewarpingConfig::Key::BOTTOM_DISTORSION_FACTOR, 1);
     m_dewarpingConfig->setValue(DewarpingConfig::Key::FISH_EYE_ANGLE, 220);
-    m_dewarpingConfig->setValue(DewarpingConfig::Key::DETECTION_DEWARPING_COUNT, 4);
+    m_dewarpingConfig->setValue(DewarpingConfig::Key::DETECTION_DEWARPING_COUNT, 5);
 
     m_videoInputConfig->setValue(VideoConfig::Key::FPS, 20);
     m_videoInputConfig->setValue(VideoConfig::Key::WIDTH, 2880);

steno/src/model/stream/video/detection/detection_thread.cpp

@@ -7,6 +7,11 @@
 #include "model/stream/video/dewarping/dewarping_helper.h"
 #include "model/stream/video/detection/detection_dewarp_optimizer.h"
 
+namespace
+{
+const float DETECTION_DUPLICATE_AZIMUTH_RANGE = 0.4f; 
+}
+
 namespace Model
 {
 DetectionThread::DetectionThread(std::shared_ptr<LockTripleBuffer<RGBImage>> imageBuffer, std::unique_ptr<IDetector> detector,
@@ -78,6 +83,7 @@ void DetectionThread::run()
             const Image& image = imageBuffer_->getLocked();
 
             std::vector<int> nextDewarpingAreas = detectionDewarpingOptimizer.getNextDetectionAreas();
+            std::vector<float> viewsMiddleAzimuth;
 
             // Dewarp each view, detect on each view and concatenate the results
             for (int i : nextDewarpingAreas)
@@ -96,12 +102,14 @@ void DetectionThread::run()
                     float middleAngleDiff = (2.f * math::PI) / dewarpingConfig_->detectionDewarpingCount;
 
                     SphericalAngleRect sphericalAngleRect = getAngleRectFromDewarpedImageRectangle(detection, dewarpingParams[i],
-                                                                                                detectionImage, fisheyeCenter,
-                                                                                                dewarpingConfig_->fisheyeAngle);
+                                                                                                   detectionImage, fisheyeCenter,
+                                                                                                   dewarpingConfig_->fisheyeAngle);
 
-                    if (!isInOverlappingZone(sphericalAngleRect, dewarpingConfig_->angleSpan, middleAngleDiff * i))
+                    float viewMiddleAzimuth = middleAngleDiff * i;
+                    if (!isInOverlappingZone(sphericalAngleRect, dewarpingConfig_->angleSpan, viewMiddleAzimuth))
                     {
                         detections.push_back(sphericalAngleRect);
+                        viewsMiddleAzimuth.push_back(viewMiddleAzimuth);
                     }
                 }
 
@@ -111,6 +119,8 @@ void DetectionThread::run()
                 }
             }
 
+            detections = getUniqueDetections(detections, viewsMiddleAzimuth);
+
             bool success = false;
 
             // Output the detections, if queue is full keep trying...
@@ -198,4 +208,50 @@ std::vector<DewarpingMapping> DetectionThread::getDewarpingMappings(
 
     return dewarpingMappings;
 }
+
+std::vector<SphericalAngleRect> DetectionThread::getUniqueDetections(const std::vector<SphericalAngleRect>& detections, 
+                                                                     const std::vector<float>& viewsMiddleAzimuth)
+{
+    std::vector<SphericalAngleRect> uniqueDetections;
+    std::vector<float> uniqueViewsMiddleAzimuth;
+    uniqueDetections.reserve(detections.size());
+
+    for (size_t i = 0; i < detections.size(); ++i)
+    {
+        const SphericalAngleRect& detection = detections[i];
+        bool isGoalDuplicate = false;
+
+        // Check if there is already a region within the angle thresholds
+        for (size_t j = 0; j < uniqueDetections.size(); ++j)
+        {
+            SphericalAngleRect& uniqueDetection = uniqueDetections[j];
+
+            if (detection.azimuth > uniqueDetection.azimuth - DETECTION_DUPLICATE_AZIMUTH_RANGE &&
+                detection.azimuth < uniqueDetection.azimuth + DETECTION_DUPLICATE_AZIMUTH_RANGE &&
+                detection.elevation > uniqueDetection.elevation - DETECTION_DUPLICATE_AZIMUTH_RANGE &&
+                detection.elevation < uniqueDetection.elevation + DETECTION_DUPLICATE_AZIMUTH_RANGE)
+            {
+                float uniqueViewMiddleAzimuth = uniqueViewsMiddleAzimuth[j];
+                float viewMiddleAzimuth = viewsMiddleAzimuth[i];
+
+                if (std::abs(detection.azimuth - viewMiddleAzimuth) < std::abs(uniqueDetection.azimuth - uniqueViewMiddleAzimuth))
+                {
+                    uniqueDetection = detection;
+                }
+
+                isGoalDuplicate = true;
+                break;
+            }
+        }
+
+        // If this region is not a duplicate, add it to the unique regions
+        if (!isGoalDuplicate)
+        {
+            uniqueDetections.emplace_back(detection.azimuth, detection.elevation, detection.azimuthSpan, detection.elevationSpan);
+            uniqueViewsMiddleAzimuth.push_back(viewsMiddleAzimuth[i]);
+        }
+    }
+
+    return uniqueDetections;
+}
 }    // namespace Model

steno/src/model/stream/video/detection/detection_thread.h

@@ -33,6 +33,8 @@ class DetectionThread : public Thread, public Subject
     std::vector<ImageFloat> getDetectionImages(const Dim2<int>& dim, int dewarpCount);
     std::vector<DewarpingMapping> getDewarpingMappings(const std::vector<DewarpingParameters>& paramsVector,
                                                        const Dim2<int>& src, const Dim2<int>& dst, int dewarpCount);
+    std::vector<SphericalAngleRect> getUniqueDetections(const std::vector<SphericalAngleRect>& detections, 
+                                                        const std::vector<float>& viewsMiddleAzimuth);
 
     std::shared_ptr<LockTripleBuffer<RGBImage>> imageBuffer_;
     std::unique_ptr<IDetector> detector_;

steno/src/model/stream/video/virtualcamera/virtual_camera_manager.cpp

@@ -76,22 +76,19 @@ void VirtualCameraManager::updateVirtualCameras(int elapsedTimeMs)
     setVirtualCameras(virtualCameras);
 }
 
-void VirtualCameraManager::updateVirtualCamerasGoal(const std::vector<SphericalAngleRect>& goals)
+void VirtualCameraManager::updateVirtualCamerasGoal(std::vector<SphericalAngleRect> goals)
 {
     if (goals.empty()) return;
 
-    // Get a vector of non duplicate goals (duplicates are within a set threshold)
-    std::vector<SphericalAngleRect> uniqueGoals = getUniqueRegions(goals);
-
     // Update the goals elevation and spans to be within bounds
-    updateRegionsBounds(uniqueGoals);
+    updateRegionsBounds(goals);
 
     // Get the distance between each virtual camera and each goal in a sorted map (map values are indices pair)
-    const std::multimap<float, std::pair<int, int>> orderedDistanceMap = getVcToGoalOrderedDistances(uniqueGoals);
+    const std::multimap<float, std::pair<int, int>> orderedDistanceMap = getVcToGoalOrderedDistances(goals);
 
     // Create vectors to keep track of which virtual camera and goal were matched
     const int vcCount = virtualCameras_.size();
-    const int goalCount = uniqueGoals.size();
+    const int goalCount = goals.size();
     std::vector<bool> isVcUnmatchedVec(vcCount, true);
     std::vector<bool> isGoalUnmatchedVec(goalCount, true);
 
@@ -113,7 +110,7 @@ void VirtualCameraManager::updateVirtualCamerasGoal(const std::vector<SphericalA
             ++matchedGoalCount;
 
             // Update the virtual camera goal and reset its time to live
-            virtualCameras_[vcIndex].goal = uniqueGoals[goalIndex];
+            virtualCameras_[vcIndex].goal = goals[goalIndex];
             virtualCameras_[vcIndex].timeToLiveMs = VIRTUAL_CAMERA_TIME_TO_LIVE_MS;
 
             // If all virtual cameras or all goals were matched, no reason to keep going
@@ -131,7 +128,7 @@ void VirtualCameraManager::updateVirtualCamerasGoal(const std::vector<SphericalA
         {
             if (isGoalUnmatchedVec[i])
             {
-                virtualCameras_.emplace_back(uniqueGoals[i], VIRTUAL_CAMERA_TIME_TO_LIVE_MS);
+                virtualCameras_.emplace_back(goals[i], VIRTUAL_CAMERA_TIME_TO_LIVE_MS);
             }
         }
     }
@@ -171,38 +168,6 @@ float VirtualCameraManager::getElevationOverflow(float elevation, float elevatio
     return elevationOverflow;
 }
 
-std::vector<SphericalAngleRect> VirtualCameraManager::getUniqueRegions(const std::vector<SphericalAngleRect>& regions)
-{
-    std::vector<SphericalAngleRect> uniqueRegions;
-    uniqueRegions.reserve(regions.size());
-
-    for (const SphericalAngleRect& region : regions)
-    {
-        bool isGoalDuplicate = false;
-
-        // Check if there is already a region within the angle thresholds
-        for (const SphericalAngleRect& uniqueRegion : uniqueRegions)
-        {
-            if (region.azimuth > uniqueRegion.azimuth - DUPLICATE_FACE_ANGLE_RANGE &&
-                region.azimuth < uniqueRegion.azimuth + DUPLICATE_FACE_ANGLE_RANGE &&
-                region.elevation > uniqueRegion.elevation - DUPLICATE_FACE_ANGLE_RANGE &&
-                region.elevation < uniqueRegion.elevation + DUPLICATE_FACE_ANGLE_RANGE)
-            {
-                isGoalDuplicate = true;
-                break;
-            }
-        }
-
-        // If this region is not a duplicate, add it to the unique regions
-        if (!isGoalDuplicate)
-        {
-            uniqueRegions.emplace_back(region.azimuth, region.elevation, region.azimuthSpan, region.elevationSpan);
-        }
-    }
-
-    return uniqueRegions;
-}
-
 void VirtualCameraManager::updateRegionsBounds(std::vector<SphericalAngleRect>& regions)
 {
     for (SphericalAngleRect& region : regions)

steno/src/model/stream/video/virtualcamera/virtual_camera_manager.h

@@ -17,14 +17,13 @@ class VirtualCameraManager : public IVirtualCameraSource
     VirtualCameraManager(float aspectRatio, float srcImageMinElevation, float srcImageMaxElevation);
 
     void updateVirtualCameras(int elapsedTimeMs);
-    void updateVirtualCamerasGoal(const std::vector<SphericalAngleRect>& goals);
+    void updateVirtualCamerasGoal(std::vector<SphericalAngleRect> goals);
     void clearVirtualCameras();
 
     std::vector<VirtualCamera> getVirtualCameras() override;
 
    private:
     float getElevationOverflow(float elevation, float elevationSpan);
-    std::vector<SphericalAngleRect> getUniqueRegions(const std::vector<SphericalAngleRect>& regions);
     void updateRegionsBounds(std::vector<SphericalAngleRect>& regions);
     std::multimap<float, std::pair<int, int>> getVcToGoalOrderedDistances(
         const std::vector<SphericalAngleRect>& targets);