Home>

I want to do Masking using OpenCV.

environment

・ Opencv4.5.x
· Java 14
・ Ubuntu 20.04 LTS
・ Mask r-cnn

Model and data to use

・ Mask_rcnn_inception_v2_coco_2018_01_28/frozen_inference_graph.pb
・ Mask_rcnn_inception_v2_coco_2018_01_28.pbtxt
・ Colors.txt
・ Mscoco_labels.names

Reference site

https://www.pyimagesearch.com/2018/11/19/mask-r-cnn-with-opencv
etc

Contents of question

I am trying to perform mask processing using opencv in Java.

(1) I want to convert detection_masks, which are acquired after executing net # forward, to 15 * 15 mask images.
In the c ++ sample, it seems that the corresponding data is acquired based on the value of classId and the Mat data is created.
How should I implement it in Java? I don't know how to convert it, can you tell me?

(2) Mat data is 100 * 90 * 15 * 15 * CV_32FC1.
Is it okay to recognize 100 mask data, 90class classification, and 15 * 15 size?

In the sample in c ++, the question part is as follows.

  for (int i = 0;i<numDetections;++ i)
    {
        float score = outDetections.at<float>(i, 2);
        if (score>confThreshold)
        {
            // Extract the bounding box
            int classId = static_cast<int>(outDetections.at<float>(i, 1));
            int left = static_cast<int>(frame.cols * outDetections.at<float>(i, 3));
            int top = static_cast<int>(frame.rows * outDetections.at<float>(i, 4));
            int right = static_cast<int>(frame.cols * outDetections.at<float>(i, 5));
            int bottom = static_cast<int>(frame.rows * outDetections.at<float>(i, 6));
            left = max (0, min (left, frame.cols --1));
            top = max (0, min (top, frame.rows --1));
            right = max (0, min (right, frame.cols -1));
            bottom = max (0, min (bottom, frame.rows --1));
            Rect box = Rect (left, top, right --left + 1, bottom --top + 1);// Extract the mask for the object Question
            Mat object Mask (outMasks.size [2],

 outMasks.size [3],

 CV_32F, outMasks.ptr<float>(i, classId));
            // Draw bounding box, colorize and show the mask on the image
            drawBox (frame, classId, score, box, objectMask);
        }
}

Thank you.

Implementation in Java
// NCHW
Mat blobImage = Dnn.blobFromImage (orgImage);
net.setInput (blobImage);
List<Mat>outputBlobs = new ArrayList<>();
List<String>outputNames = Arrays.asList (new String [] {"detection_out_final", "detection_masks"});
net.forward (outputBlobs, outputNames);
Mat detectionOutFinal = outputBlobs.get (0);
Mat detectionMasks = outputBlobs.get (1);// Value that I am asking about in this case
// Output size of masks is NxCxHxW where
// N --number of detected boxes
// C --number of classes (excluding background)
// HxW --segmentation shape
// detections.size (3) == 7
Mat detection = detectionOutFinal.reshape (1, (int) detectionOutFinal.total ()/detectionOutFinal.size (3));
// detections.size (2) == 100
int numDetections = detectionOutFinal.size (2);
// masks.size (1) == 90
int numClasses = detectionMasks.size (1);
int thickness = 3;
double threshold = 0.5;
int height = orgImage.height ();
int width = orgImage.width ();
Mat dstImage = orgImage.clone ();
Scalar color = new Scalar (0, 255, 0);
IntStream.range (0, numDetections) .filter (i->detection.get (i, 2) [0]>threshold) .forEach (i->{
    int classId = (int) detection.get (i, 1) [0];
    String className = names.get (classId);double score = detection.get (i, 2) [0];
    double left = detection.get (i, 3) [0] * width;
    double top = detection.get (i, 4) [0] * height;
    double right = detection.get (i, 5) [0] * width;
    double bottom = detection.get (i, 6) [0] * height;
    left = Math.max (0, Math.min (left, height --thickness));
    top = Math.max (0, Math.min (top, width --thickness));
    right = Math.max (0, Math.min (right, height --thickness));
    bottom = Math.max (0, Math.min (bottom, width --thickness));
    System.out.println (score + "" + classId + "" + className + "" + left + "" + top + "" + right + "" + bottom);
        Rect rect = new Rect (new double [] {left, top, right --left + 1, bottom --top + 1});
    Imgproc.rectangle (dstImage, rect, color, thickness);
});
HighGui.imshow ("Input Image", dst Image);
# mask_rcnn_inception_v2_coco_2018_01_28.pbtxt
Abbreviation
5928 node {
5929 name: "detection_out_final"
5930 op: "DetectionOutput"
5931 input: "variance_encoded/flatten"
5932 input: "SecondStageBoxPredictor/Reshape_1/Reshape"
5933 input: "detection_out/slice/reshape"
5934 attr {
5935 key: "background_label_id"
5936 value {
5937 i: 91
5938}
5939}
‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥
7577 node {
7578 name: "detection_masks"
7579 op: "Sigmoid"
7580 input: "SecondStageBoxPredictor_1/transpose"
7581}
‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥‥
What I tried

detectionMasks.get (i, classId)
etc...

  • Answer # 1

    Mat detection = detectionOutFinal.reshape (1, (int) detectionOutFinal.total ()/detectionOutFinal.size (3));
    int numDetections = detectionOutFinal.size (2);
    Mat reshapeMask = detectionMasks.reshape (1,
    (int) detectionMasks.total ()/(detectionMasks.size (2) * detectionMasks.size (3)));

    IntStream.range (0, numDetections) .filter (i->detection.get (i, 2) [0]>threshold) .forEach (i->{
    int classId = (int) detection.get (i, 1) [0];
    // color from txt
    Scalar color = coloers.get (classId% coloers.size ());
    // axis
    Rect box = createRect (detection, i, cols, rows);
    // for rectangle txt
    String label = createLabel (detection, i, cols, rows, names);
    // it will be ractangle on image
    rectangleProcessing (dstImage, label, color, box);
    // 1 x 255
    Mat objectMask = reshapeMask.row (i * detectionMasks.size (0) + classId);
    // 15 x 15
    objectMask = objectMask.reshape (1, detectionMasks.size (2));
    // it will be mask on image
    Mat maskImage = createMask (objectMask, dstImage.size (), color, box);
    maskImages.add (maskImage);
    maskProcessing (dstImage, maskImage, box);
    });

    private Mat createMask (Mat objectMask, Size maskSize, Scalar maskColor, Rect box) {
    Imgproc.resize (objectMask, objectMask, new Size (box.width + 1, box.height + 1));

    Mat maskImage = new Mat ();
    objectMask.convertTo (maskImage, CvType.CV_8U);

    Mat dstMaskImage = Mat.zeros (maskSize, CvType.CV_8UC3);
    Listcontours = new ArrayList<>();
    Mat hierarchy = new Mat ();

    Imgproc.findContours (maskImage, contours, hierarchy, Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE);
    Imgproc.drawContours (dstMaskImage, contours, -1, maskColor, Imgproc.FILLED, Imgproc.LINE_8, hierarchy, 100,
    new Point (box.x, box.y));

    return dstMaskImage;
    }