add mask syncronization strategy for DynamicExtractor

Horacehxw · May 11, 2019 · 8183f83 · 8183f83
1 parent 9a5acc1
commit 8183f83
Show file tree

Hide file tree

Showing 30 changed files with 16,425 additions and 23 deletions.
diff --git a/Examples/DynamicExtractor/back.png b/Examples/DynamicExtractor/back.png
diff --git a/Examples/DynamicExtractor/back2.png b/Examples/DynamicExtractor/back2.png
diff --git a/Examples/DynamicExtractor/outimg_1.png b/Examples/DynamicExtractor/outimg_1.png
diff --git a/Examples/DynamicExtractor/outimg_2.png b/Examples/DynamicExtractor/outimg_2.png
diff --git a/Examples/DynamicExtractor/outimg_3.png b/Examples/DynamicExtractor/outimg_3.png
diff --git a/Examples/DynamicExtractor/outimg_false.png b/Examples/DynamicExtractor/outimg_false.png
diff --git a/Examples/DynamicExtractor/test.cc b/Examples/DynamicExtractor/test.cc
@@ -51,6 +51,7 @@ void draw_feature(Mat& img_1, Mat& mask) {
     imwrite("Examples/DynamicExtractor/outimg_3.png", outimg);
 
     drawKeypoints( greyimg1, keypoints_4, outimg, Scalar(0,0,255), DrawMatchesFlags::DEFAULT);
+    drawKeypoints( outimg, keypoints_1, outimg, Scalar(0,255,0), DrawMatchesFlags::DEFAULT);
     imwrite("Examples/DynamicExtractor/outimg_false.png", outimg);
 }
 
@@ -59,7 +60,7 @@ int main() {
             "ModelsCNN/"
             );
 
-    Mat frame = imread("Examples/DynamicExtractor/human.png", CV_LOAD_IMAGE_COLOR);
+    Mat frame = imread("Examples/DynamicExtractor/back.png", CV_LOAD_IMAGE_COLOR);
     Mat mask;
 
     std::chrono::steady_clock::time_point t1 = std::chrono::steady_clock::now();

diff --git a/Examples/MaskRCNN/dynamic_mask.png b/Examples/MaskRCNN/dynamic_mask.png
diff --git a/Examples/MaskRCNN/mask.png b/Examples/MaskRCNN/mask.png
diff --git a/Examples/MaskRCNN/mask_rcnn.cc b/Examples/MaskRCNN/mask_rcnn.cc
@@ -286,7 +286,7 @@ int main(int argc, char *argv[])
     VideoWriter video;
     Mat frame, blob;
     //frame = imread("human.png", CV_LOAD_IMAGE_COLOR);
-    cap.open("./human.png");
+    cap.open("./lqh.jpg");
 
 
     // Create a window

diff --git a/Examples/RGB-D/rgbd_tum.cc b/Examples/RGB-D/rgbd_tum.cc
@@ -35,9 +35,9 @@ void LoadImages(const string &strAssociationFilename, vector<string> &vstrImageF
                 vector<string> &vstrImageFilenamesD, vector<double> &vTimestamps);
 
 int main(int argc, char **argv) {
-    if (argc != 6 && argc != 7) {
+    if (argc != 6 && argc != 7 && argc != 8) {
         cerr << endl
-             << "Usage: ./rgbd_tum path_to_vocabulary path_to_settings path_to_sequence path_to_association target_name path_to_cnn_model"
+             << "Usage: ./rgbd_tum path_to_vocabulary path_to_settings path_to_sequence path_to_association target_name path_to_cnn_model mask_sync"
              << endl;
         return 1;
     }
@@ -61,15 +61,19 @@ int main(int argc, char **argv) {
 
     // Create SLAM system. It initializes all system threads and gets ready to process frames.
     string model_path;
+    int skip = 1;
     if (argc > 6) {
         model_path = string(argv[6]);
     }
+    if (argc > 7) {
+        skip = stoi(string(argv[7]));
+    }
 
     // test 5 runs!
 
     {
         //cout << "run number " << runs << endl;
-        ORB_SLAM2::System SLAM(argv[1], argv[2], ORB_SLAM2::System::RGBD, true, model_path);
+        ORB_SLAM2::System SLAM(argv[1], argv[2], ORB_SLAM2::System::RGBD, true, model_path, skip);
 
 
         // Vector for tracking time statistics

diff --git a/dynamic_mask.png b/dynamic_mask.png
diff --git a/include/DynamicExtractor.h b/include/DynamicExtractor.h
@@ -26,24 +26,30 @@ namespace ORB_SLAM2 {
     public:
         // need model location to load the model
         // only supports mask-rcnn for now
-        DynamicExtractor(const std::string &strModelPath,
+        DynamicExtractor(const std::string &strModelPath, int maxUsage=1,
                          float confThreshold = 0.5, float maskThreshold = 0.3);
 
         // compute dynamic mask for given frame
         void extractMask(const cv::Mat &frame, cv::Mat &dynamic_mask);
 
     private:
+        // do real mask extraction under CNN
+        cv::Mat extractMask(const cv::Mat &frame);
         // return non-zero if the corresponding class is dynamic
         bool is_dynamic(int classId) {
             return dynamicClasses.count(classes[classId]);
         }
 
         float confThreshold; // Confidence threshold
         float maskThreshold; // Mask threshold
+        int maskUsage; // prevMask usage counter
+        int maxUsage; // max number of mask synchronization
 
         std::vector<std::string> classes; // classId --> className
         std::unordered_set<std::string> dynamicClasses; // name of dynamic classes
         cv::dnn::Net net; // mask-rcnn model
+        cv::Mat prevMask;
+
     };
 
 }

diff --git a/include/ORBextractor.h b/include/ORBextractor.h
@@ -50,7 +50,7 @@ class ORBextractor
     enum {HARRIS_SCORE=0, FAST_SCORE=1 };
 
     ORBextractor(int nfeatures, float scaleFactor, int nlevels,
-                 int iniThFAST, int minThFAST, const std::string &strModelPath);
+                 int iniThFAST, int minThFAST, const std::string &strModelPath, int skip);
 
     ~ORBextractor(){}
 

diff --git a/include/System.h b/include/System.h
@@ -61,7 +61,7 @@ class System
 
     // Initialize the SLAM system. It launches the Local Mapping, Loop Closing and Viewer threads.
     System(const string &strVocFile, const string &strSettingsFile, const eSensor sensor, const bool bUseViewer = true,
-            const string &strModelPath="");
+            const string &strModelPath="", int skip=1);
 
     // Proccess the given stereo frame. Images must be synchronized and rectified.
     // Input images: RGB (CV_8UC3) or grayscale (CV_8U). RGB is converted to grayscale.

diff --git a/include/Tracking.h b/include/Tracking.h
@@ -56,7 +56,7 @@ class Tracking
 public:
     Tracking(System *pSys, ORBVocabulary *pVoc, FrameDrawer *pFrameDrawer, MapDrawer *pMapDrawer, Map *pMap,
              KeyFrameDatabase *pKFDB, const string &strSettingPath, const int sensor,
-             const string &strModelPath);
+             const string &strModelPath, int skip=1);
 
     // Preprocess the input and call Track(). Extract features and performs stereo matching.
     cv::Mat GrabImageStereo(const cv::Mat &imRectLeft,const cv::Mat &imRectRight, const double &timestamp);