iOS11+Flutter Objective-C Text Rectangle Detection

TSTextRectangleDetector.m

#import "TSTextRectangleDetector.h"
#import <CoreML/CoreML.h>
#import <Vision/Vision.h>

@implementation TSTextRectangleDetector {
  FlutterEventSink _eventSink;
  BOOL _running;
  BOOL _wasEmpty;
}

- (void)setEventChannel:(FlutterEventChannel*)eventChannel {
  _eventChannel = eventChannel;
  [_eventChannel setStreamHandler:self];
}

- (void)didReceiveSampleBuffer:(CMSampleBufferRef)sampleBuffer withPixelBuffer:(CVPixelBufferRef)pixelBuffer {
  if (!_eventSink) {
    return;
  }
  if (_running) {
    NSLog(@"TSTextRectangleDetector::didReceiveSampleBuffer:: skip, previous task is running");
    return;
  }

  if (@available(iOS 11.0, *)) {
    _running = YES;
    runRectangleDetectorCallback completion = ^(NSArray * _Nullable blocks, NSError * _Nullable error) {
      self->_running = NO;
      if (error) {
        return;
      }
      
      // For performance reason, we skip reporting if there are no rectangles repeatedly.
      if (0 < blocks.count) {
        self->_wasEmpty = NO;
      } else {
        if (self->_wasEmpty) {
          return;
        }
        self->_wasEmpty = YES;
      }
      
      if (self->_eventSink) {
        NSAssert(blocks, @"logical error, blocks must not be nil");
        // FIXME lock _eventSink since it can be nil before call it.
        //       or, ideally it can be prevent by keep the value non-nil(tweak onCancelWithArguments not to clear it).
        NSDictionary* event = @{@"eventName": @"rectangles", @"rectangles": blocks};
        self->_eventSink(event);
      }
    };
    
    [TSTextRectangleDetector runRectangleDetectorWithPixelBuffer:pixelBuffer
                                                     orientation:kCGImagePropertyOrientationUp
                                                  completion:completion];
  };
}

+ (void)runRectangleDetectorWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
                                orientation:(CGImagePropertyOrientation)orientation
                                 completion:(runRectangleDetectorCallback)completion {
  if (@available(iOS 11.0, *)) {
    CFRetain(pixelBuffer);

    VNImageRequestHandler *requestHandler = [[VNImageRequestHandler alloc] initWithCVPixelBuffer:pixelBuffer
                                                                                     orientation:kCGImagePropertyOrientationUp
                                                                                         options:@{}];
    runRectangleDetectorCallback localCompletion = ^(NSArray * _Nullable blocks, NSError * _Nullable error) {
      CFRelease(pixelBuffer);
      completion(blocks, error);
    };
    [TSTextRectangleDetector runRectangleDetectorWithRequestHandler:requestHandler completion:localCompletion];
  } else {
    completion(nil, nil);
  }
}

+ (void)runRectangleDetectorWithCGImage:(CGImageRef)cgImage
                            orientation:(CGImagePropertyOrientation)orientation
                                 completion:(runRectangleDetectorCallback)completion {
  if (@available(iOS 11.0, *)) {
    VNImageRequestHandler *requestHandler = [[VNImageRequestHandler alloc] initWithCGImage:cgImage
                                                                               orientation:kCGImagePropertyOrientationUp
                                                                                   options:@{}];
    [TSTextRectangleDetector runRectangleDetectorWithRequestHandler:requestHandler completion:completion];
  } else {
    completion(nil, nil);
  }
}

+ (void)runRectangleDetectorWithRequestHandler:(VNImageRequestHandler*)requestHandler
                                    completion:(runRectangleDetectorCallback)completion API_AVAILABLE(ios(11.0)) {
  VNRequestCompletionHandler completionHandler = ^(VNRequest * _Nonnull request, NSError * _Nullable error) {
    if (error) {
      NSLog(@"error %@", error);
      completion(nil, error);
      return;
    }
    
    NSMutableArray* blocks = [[NSMutableArray alloc] initWithCapacity:request.results.count * 4];
    for (VNTextObservation *textObservation in request.results) {
      float y = 1.0 - textObservation.boundingBox.origin.y - textObservation.boundingBox.size.height;
      [blocks addObject:[NSNumber numberWithFloat:textObservation.boundingBox.origin.x]];
      [blocks addObject:[NSNumber numberWithFloat:y]];
      [blocks addObject:[NSNumber numberWithFloat:textObservation.boundingBox.size.width]];
      [blocks addObject:[NSNumber numberWithFloat:textObservation.boundingBox.size.height]];
    }
    
    completion(blocks, nil);
  };
  
  VNDetectTextRectanglesRequest *request = [[VNDetectTextRectanglesRequest alloc] initWithCompletionHandler:completionHandler];
  NSError *error;
  [requestHandler performRequests:@[request] error:&error];
  if (error) {
    NSLog(@"%@", error);
    completion(nil, error);
  }
}

#pragma mark FlutterStreamHandler

- (FlutterError *_Nullable)onListenWithArguments:(id _Nullable)arguments
                                       eventSink:(nonnull FlutterEventSink)events {
  _eventSink = events;
  return nil;
}

- (FlutterError *_Nullable)onCancelWithArguments:(id _Nullable)arguments {
  _eventSink = nil;
  return nil;
}

@end