Tensorflow Lite using cv::Mat as input (MobileNetV2)- C++ & Python

opencv_to_tflite.cpp

/* tflite model:
 *     - input shape: (224,224,3)
 *     - output shape: (128)
 * In C++, cv::Mat should be flatten as input. If input has 3 channel,
 * then it should be RGBRGBRGB... */

#include <iostream>
#include <iomanip>

#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/model.h"
#include "tensorflow/lite/optional_debug_tools.h"

#include "opencv2/opencv.hpp"
using namespace std;
typedef cv::Point3_<float> Pixel;

const uint WIDTH = 224;
const uint HEIGHT = 224;
const uint CHANNEL = 3;
const uint OUTDIM = 128;

void normalize(Pixel &pixel){
    pixel.x = ((pixel.x / 255.0)-0.5)*2.0;
    pixel.y = ((pixel.y / 255.0)-0.5)*2.0;
    pixel.z = ((pixel.z / 255.0)-0.5)*2.0;
}

int main(){
    // read image file
    cv::Mat img = cv::imread("01.jpg");

    // convert to float; BGR -> RGB
    cv::Mat inputImg;
    img.convertTo(inputImg, CV_32FC3);
    cv::cvtColor(inputImg, inputImg, cv::COLOR_BGR2RGB);

    // normalize to -1 & 1
    Pixel* pixel = inputImg.ptr<Pixel>(0,0);
    const Pixel* endPixel = pixel + inputImg.cols * inputImg.rows;
    for (; pixel != endPixel; pixel++)
        normalize(*pixel);

    // resize image as model input
    cv::resize(inputImg, inputImg, cv::Size(WIDTH, HEIGHT));

    // create model
    std::unique_ptr<tflite::FlatBufferModel> model =
        tflite::FlatBufferModel::BuildFromFile("model.tflite");
    tflite::ops::builtin::BuiltinOpResolver resolver;
    std::unique_ptr<tflite::Interpreter> interpreter;
    tflite::InterpreterBuilder(*model.get(), resolver)(&interpreter);
    interpreter->AllocateTensors();

    // get input & output layer
    float* inputLayer = interpreter->typed_input_tensor<float>(0);
    float* outputLayer = interpreter->typed_output_tensor<float>(0);

    // flatten rgb image to input layer.
    float* inputImg_ptr = inputImg.ptr<float>(0);
    memcpy(inputLayer, inputImg.ptr<float>(0),
            WIDTH * HEIGHT * CHANNEL * sizeof(float));

    // compute model instance
    interpreter->Invoke();

    // print final result
    printf("[");
    for(int i=1; i<=OUTDIM; i++){
        cout << setw(11) << fixed << setprecision(8)
            << outputLayer[i-1] << " ";
        if(i%6==0) printf("\n ");
    }
    printf("]");
    return 0;
}

opencv_to_tflite.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-

from tflite_runtime.interpreter import Interpreter
import numpy as np
import cv2


if __name__ == "__main__":
    img = cv2.imread("01.jpg").astype(np.float32)
    img = cv2.resize(((img/255.0)-0.5)*2.0, (224, 224))
    img = img[:,:,::-1]

    mobilePath = "model.tflite"
    model = Interpreter(model_path=mobilePath)
    model.allocate_tensors()

    inputLayer = model.get_input_details()[0]["index"]
    outputLayer = model.get_output_details()[0]["index"]

    model.set_tensor(inputLayer, [img])
    model.invoke()
    print(model.get_tensor(outputLayer)[0])

C++ compile makefile

# prerequisites:
#   - opencv lib
#   - tenosorflow lite lib
#
# Test on Raspberry Pi 4
#   - Raspbian GNU/Linux 10 (buster)
#   - 5.4.51-v7l+

CC := arm-linux-gnueabihf-g++
EXE := main
OBJ := main.cpp
TF_DIR := /opt/tensorflow_src
OPENCV_DIR := /opt/opencv
OPENCV_LIB := -lopencv_gapi -lopencv_stitching -lopencv_aruco -lopencv_bgsegm \
	-lopencv_bioinspired -lopencv_ccalib -lopencv_dnn_objdetect \
	-lopencv_dnn_superres -lopencv_dpm -lopencv_face -lopencv_freetype \
	-lopencv_fuzzy -lopencv_hfs -lopencv_img_hash -lopencv_intensity_transform \
	-lopencv_line_descriptor -lopencv_mcc -lopencv_quality -lopencv_rapid \
	-lopencv_reg -lopencv_rgbd -lopencv_saliency -lopencv_stereo \
	-lopencv_structured_light -lopencv_phase_unwrapping -lopencv_superres \
	-lopencv_optflow -lopencv_surface_matching -lopencv_tracking \
	-lopencv_highgui -lopencv_datasets -lopencv_text -lopencv_plot \
	-lopencv_videostab -lopencv_videoio -lopencv_xfeatures2d -lopencv_shape \
	-lopencv_ml -lopencv_ximgproc -lopencv_video -lopencv_dnn \
	-lopencv_xobjdetect -lopencv_objdetect -lopencv_calib3d -lopencv_imgcodecs \
	-lopencv_features2d -lopencv_flann -lopencv_xphoto -lopencv_photo \
	-lopencv_imgproc -lopencv_core -littnotify -llibprotobuf -llibwebp \
	-llibopenjp2 -lIlmImf -lquirc -ltegra_hal -lade -ljpeg -lpng -ltiff \
	-lz -ldl -lm -lpthread -lrt
all:
	$(CC) -Bstatic -std=c++11 -o $(EXE) $(OBJ) \
	-I$(TF_DIR)/tensorflow/lite/tools/make/downloads/flatbuffers/include \
	-I$(TF_DIR) \
	-L$(TF_DIR)/tensorflow/lite/tools/make/gen/rpi_armv7l/lib \
	-ltensorflow-lite \
	-I$(OPENCV_DIR)/include/opencv4 \
	-L$(OPENCV_DIR)/lib/ \
	-L$(OPENCV_DIR)/lib/opencv4/3rdparty \
	-L$(OPENCV_DIR)/build/lib \
	$(OPENCV_LIB)
	# `pkg-config --cflags --libs --static opencv4`
	@echo "Compile Complete"

clean:
	rm $(EXE)