Process camera image in flutter

Question

I'm trying to process the camera image input in Flutter but I can't seem to get it to work.

I have a listener on the camera feed that launch a function that is supposed to process the images btu everytime I end up doing anything in the function the program interface freeze. My guess is that I have too many frames to process and thus the program freezes but I have no idea how to ignore old frames.

Here is my code so far

import 'package:flutter/material.dart';
import 'package:camera/camera.dart';
import 'package:image/image.dart' as imglib;
import 'package:flutter/scheduler.dart';
import 'user_data_container.dart';


class Camera extends StatefulWidget {

  Camera();

  @override
  _CameraState createState() => new _CameraState();
}

class _CameraState extends State<Camera> {
  CameraController controller;
  bool isDetecting = false;
  double redavg;
  imglib.Image last_image;
  int count = 0;

  Image _image_display;

  @override
  void initState() {
    super.initState();
    SchedulerBinding.instance.addPostFrameCallback((_) {
      _initializeApp();
    });
  }

  void _initializeApp() async {
    var cameras = UserDataContainer.of(context).data.cameras;
    if (cameras == null || cameras.length < 1) {
      print('No camera is found');
    } else {
      controller = new CameraController(
        cameras[0],
        ResolutionPreset.high,
      );
      controller.initialize().then((_) {
        if (!mounted) {
          return;
        }
        setState(() {});

        controller.startImageStream((CameraImage img) async {
          print(count.toString() + "Stream. detecting: " + isDetecting.toString());
          if (!isDetecting) {
            isDetecting = true;

            int startTime = new DateTime.now().millisecondsSinceEpoch;
            _processCameraImage(img);
//            int endTime = new DateTime.now().millisecondsSinceEpoch;

//            print("Detection done in " + (endTime - startTime).toString());
//            isDetecting = false;
          }
          else{
            print("It's detecting");
          }
        });
      });
    }
  }

  @override
  void dispose() {
    controller?.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    if (controller == null || !controller.value.isInitialized) {
      return Container();
    }


    return

      Scaffold(
        body: Column(
          children: <Widget>[
            Text("red: "+ redavg.toString() + " "+ count.toString()),
            Expanded(child: last_image == null ? Container() : _image_display),
          ],
        ),
      );

//    );
  }

  void _processCameraImage(CameraImage image) async {
    count = count +1;
    print("p: " + image.planes.length.toString());
//    if (isDetecting) {
//      print("Already detecting;");
//      return;
//    }
//    else{
//      isDetecting = true;
      try {
        await processFrame(image);
      } catch (e) {
        // await handleExepction(e)
      } finally {
        print("Done detecting :)");
//        isDetecting = false;
      }
//    }



  }

  void processFrame(CameraImage image) async {
    print ("convert");
    imglib.Image convertedImage = await _convertCameraImage(image);
    last_image = convertedImage;

    imglib.PngEncoder pngEncoder = new imglib.PngEncoder(level: 0, filter: 0);
    // Convert to png
    List<int> png = pngEncoder.encodeImage(last_image);
    _image_display = Image.memory(png);


//    var colors = colorAverage(convertedImage);
    setState(() {
//      redavg = colors[2];
//      print(colors.toString());
    });
  }


  imglib.Image _convertCameraImage(CameraImage image) {
    int width = image.width;
    int height = image.height;
    var img = imglib.Image(image.planes[0].bytesPerRow, height); // Create Image buffer
    const int hexFF = 0xFF000000;
    final int uvyButtonStride = image.planes[1].bytesPerRow;
    final int uvPixelStride = image.planes[1].bytesPerPixel;
    for (int x = 0; x < width; x++) {
      for (int y = 0; y < height; y++) {
        final int uvIndex =
            uvPixelStride * (x / 2).floor() + uvyButtonStride * (y / 2).floor();
        final int index = y * width + x;
        final yp = image.planes[0].bytes[index];
        final up = image.planes[1].bytes[uvIndex];
        final vp = image.planes[2].bytes[uvIndex];
        // Calculate pixel color
        int r = (yp + vp * 1436 / 1024 - 179).round().clamp(0, 255);
        int g = (yp - up * 46549 / 131072 + 44 - vp * 93604 / 131072 + 91)
            .round()
            .clamp(0, 255);
        int b = (yp + up * 1814 / 1024 - 227).round().clamp(0, 255);
        // color: 0x FF  FF  FF  FF
        //           A   B   G   R
        img.data[index] = hexFF | (b << 16) | (g << 8) | r;
      }
    }
    // Rotate 90 degrees to upright
//    var img1 = imglib.copyRotate(img, 90);

    imglib.PngEncoder pngEncoder = new imglib.PngEncoder(level: 0, filter: 0);
    // Convert to png
    List<int> png = pngEncoder.encodeImage(img);
    _image_display = Image.memory(png);

    return img;
  }

}

Answer 1

I know its a little bit late, but you can use multi threading to process the image in different thread. For example you can call your function like this:

try {
   // await processFrame(image);
  final result = await compute(convertCameraImage, image);
  setState({
    _image_display = Image.memory(png);
  });
} catch (e) {
  // await handleExepction(e)
}

Modified processFrame Function:

List<int> convertCameraImage(CameraImage image) {
    int width = image.width;
    int height = image.height;
    var img = imglib.Image(image.planes[0].bytesPerRow, height); // Create Image buffer
    const int hexFF = 0xFF000000;
    final int uvyButtonStride = image.planes[1].bytesPerRow;
    final int uvPixelStride = image.planes[1].bytesPerPixel;
    for (int x = 0; x < width; x++) {
      for (int y = 0; y < height; y++) {
        final int uvIndex =
            uvPixelStride * (x / 2).floor() + uvyButtonStride * (y / 2).floor();
        final int index = y * width + x;
        final yp = image.planes[0].bytes[index];
        final up = image.planes[1].bytes[uvIndex];
        final vp = image.planes[2].bytes[uvIndex];
        // Calculate pixel color
        int r = (yp + vp * 1436 / 1024 - 179).round().clamp(0, 255);
        int g = (yp - up * 46549 / 131072 + 44 - vp * 93604 / 131072 + 91)
            .round()
            .clamp(0, 255);
        int b = (yp + up * 1814 / 1024 - 227).round().clamp(0, 255);
        // color: 0x FF  FF  FF  FF
        //           A   B   G   R
        img.data[index] = hexFF | (b << 16) | (g << 8) | r;
      }
    }
    // Rotate 90 degrees to upright
//    var img1 = imglib.copyRotate(img, 90);

    imglib.PngEncoder pngEncoder = new imglib.PngEncoder(level: 0, filter: 0);
    // Convert to png
    List<int> png = pngEncoder.encodeImage(img);
    // _image_display = Image.memory(png);

    return png;
  }

The reason your ui freezes is because the process image function is running on the main thread and it takes up too much processing. If you run the function in different thread it wont freeze your ui and the process will happen in background.

For better understanding about compute you can use this link: https://api.flutter.dev/flutter/foundation/compute-constant.html