Merge pull request #12697 from tellowkrinkle:FasterCocoaWindows

* Make cocoa windows draw faster

* Use a CALayer for rendering when possible Uses GPU to scale images, which is important because retina macs will want window sizes much larger (in pixels) than the image

* Fix mouse logic for cocoa windows

* Only halve resolution on retina if image is larger than display

modules/highgui/src/window_cocoa.mm

@@ -74,7 +74,6 @@ CV_IMPL int cvWaitKey (int maxWait) {return 0;}
 
 #include <iostream>
 
-const int TOP_BORDER  = 7;
 const int MIN_SLIDER_WIDTH=200;
 
 static NSApplication *application = nil;
@@ -82,10 +81,10 @@ static NSAutoreleasePool *pool = nil;
 static NSMutableDictionary *windows = nil;
 static bool wasInitialized = false;
 
-@interface CVView : NSView {
+@interface CVView : NSView
-    NSImage *image;
+@property(retain) NSView *imageView;
-}
 @property(retain) NSImage *image;
+@property int sliderHeight;
 - (void)setImageData:(CvArr *)arr;
 @end
 
@@ -221,32 +220,38 @@ CV_IMPL void cvShowImage( const char* name, const CvArr* arr)
     if(window)
     {
         bool empty = [[window contentView] image] == nil;
-        NSRect rect = [window frame];
         NSRect vrectOld = [[window contentView] frame];
-
+        NSSize oldImageSize = [[[window contentView] image] size];
         [[window contentView] setImageData:(CvArr *)arr];
         if([window autosize] || [window firstContent] || empty)
         {
-            //Set new view size considering sliders (reserve height and min width)
+            NSSize imageSize = [[[window contentView] image] size];
-            NSRect vrectNew = vrectOld;
+            // Only adjust the image size if the new image is a different size from the previous
-            int slider_height = 0;
+            if (oldImageSize.height != imageSize.height || oldImageSize.width != imageSize.width)
-            if ([window respondsToSelector:@selector(sliders)]) {
+            {
-                for(NSString *key in [window sliders]) {
+                //Set new view size considering sliders (reserve height and min width)
-                    slider_height += [[[window sliders] valueForKey:key] frame].size.height;
+                NSSize scaledImageSize;
+                if ([[window contentView] respondsToSelector:@selector(convertSizeFromBacking:)])
+                {
+                    // Only resize for retina displays if the image is bigger than the screen
+                    NSSize screenSize = NSScreen.mainScreen.visibleFrame.size;
+                    CGFloat titleBarHeight = window.frame.size.height - [window contentRectForFrameRect:window.frame].size.height;
+                    screenSize.height -= titleBarHeight;
+                    if (imageSize.width > screenSize.width || imageSize.height > screenSize.height)
+                    {
+                        scaledImageSize = [[window contentView] convertSizeFromBacking:imageSize];
+                    }
                 }
+                else
+                {
+                    scaledImageSize = imageSize;
+                }
+                NSSize contentSize = vrectOld.size;
+                contentSize.height = scaledImageSize.height + [window contentView].sliderHeight;
+                contentSize.width = std::max<int>(scaledImageSize.width, MIN_SLIDER_WIDTH);
+                [window setContentSize:contentSize]; //adjust sliders to fit new window size
             }
-            vrectNew.size.height = [[[window contentView] image] size].height + slider_height;
-            vrectNew.size.width = std::max<int>([[[window contentView] image] size].width, MIN_SLIDER_WIDTH);
-            [[window contentView]  setFrameSize:vrectNew.size]; //adjust sliders to fit new window size
-
-            rect.size.width += vrectNew.size.width - vrectOld.size.width;
-            rect.size.height += vrectNew.size.height - vrectOld.size.height;
-            rect.origin.y -= vrectNew.size.height - vrectOld.size.height;
-
-            [window setFrame:rect display:YES];
         }
-        else
-            [window display];
         [window setFirstContent:NO];
     }
     [localpool drain];
@@ -259,10 +264,9 @@ CV_IMPL void cvResizeWindow( const char* name, int width, int height)
     NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init];
     CVWindow *window = cvGetWindow(name);
     if(window && ![window autosize]) {
-        NSRect frame = [window frame];
+        height += [window contentView].sliderHeight;
-        frame.size.width = width;
+        NSSize size = { width, height };
-        frame.size.height = height;
+        [window setContentSize:size];
-        [window setFrame:frame display:YES];
     }
     [localpool drain];
 }
@@ -532,7 +536,7 @@ CV_IMPL int cvNamedWindow( const char* name, int flags )
     NSScreen* mainDisplay = [NSScreen mainScreen];
 
     NSString *windowName = [NSString stringWithFormat:@"%s", name];
-    NSUInteger showResize = (flags == CV_WINDOW_AUTOSIZE) ? 0: NSResizableWindowMask ;
+    NSUInteger showResize = NSResizableWindowMask;
     NSUInteger styleMask = NSTitledWindowMask|NSMiniaturizableWindowMask|showResize;
     CGFloat windowWidth = [NSWindow minFrameWidthWithTitle:windowName styleMask:styleMask];
     NSRect initContentRect = NSMakeRect(0, 0, windowWidth, 0);
@@ -728,6 +732,22 @@ void cv::setWindowTitle(const String& winname, const String& title)
     [localpool drain];
 }
 
+static NSSize constrainAspectRatio(NSSize base, NSSize constraint) {
+    CGFloat heightDiff = (base.height / constraint.height);
+    CGFloat widthDiff = (base.width / constraint.width);
+    if (widthDiff == heightDiff) {
+        return base;
+    }
+    else if (widthDiff > heightDiff) {
+        NSSize out = { constraint.width / constraint.height * base.height, base.height };
+        return out;
+    }
+    else {
+        NSSize out = { base.width, constraint.height / constraint.width * base.width };
+        return out;
+    }
+}
+
 @implementation CVWindow
 
 @synthesize mouseCallback;
@@ -743,22 +763,19 @@ void cv::setWindowTitle(const String& winname, const String& title)
     NSPoint mp = [NSEvent mouseLocation];
     //NSRect visible = [[self contentView] frame];
     mp = [self convertScreenToBase: mp];
-    double viewHeight = [self contentView].frame.size.height;
+    CVView *contentView = [self contentView];
-    double viewWidth = [self contentView].frame.size.width;
+    NSSize viewSize = contentView.frame.size;
-    CVWindow *window = (CVWindow *)[[self contentView] window];
+    if (contentView.imageView) {
-    if ([window respondsToSelector:@selector(sliders)]) {
+        viewSize = contentView.imageView.frame.size;
-        for(NSString *key in [window sliders]) {
-            NSSlider *slider = [[window sliders] valueForKey:key];
-            viewHeight = std::min(viewHeight, (double)([slider frame].origin.y));
-        }
     }
-    viewHeight -= TOP_BORDER;
+    else {
-    mp.y = viewHeight - mp.y;
+        viewSize.height -= contentView.sliderHeight;
+    }
+    mp.y = viewSize.height - mp.y;
 
-    NSImage* image = ((CVView*)[self contentView]).image;
+    NSSize imageSize = contentView.image.size;
-    NSSize imageSize = [image size];
+    mp.y *= (imageSize.height / std::max(viewSize.height, 1.));
-    mp.x = mp.x * imageSize.width / std::max(viewWidth, 1.);
+    mp.x *= (imageSize.width / std::max(viewSize.width, 1.));
-    mp.y = mp.y * imageSize.height / std::max(viewHeight, 1.);
 
     if( mp.x >= 0 && mp.y >= 0 && mp.x < imageSize.width && mp.y < imageSize.height )
         mouseCallback(type, mp.x, mp.y, flags, mouseParam);
@@ -862,17 +879,14 @@ void cv::setWindowTitle(const String& winname, const String& title)
     viewSize.width = std::max<int>(viewSize.width, MIN_SLIDER_WIDTH);
 
     // Update slider sizes
-    [[self contentView] setFrameSize:viewSize];
+    [self contentView].sliderHeight += sliderSize.height;
-    [[self contentView] setNeedsDisplay:YES];
+
+    if ([[self contentView] image] && ![[self contentView] imageView]) {
+        [[self contentView] setNeedsDisplay:YES];
+    }
 
     //update window size to contain sliders
-    NSRect rect = [self frame];
+    [self setContentSize: viewSize];
-    rect.size.height += [slider frame].size.height;
-    rect.size.width = std::max<int>(rect.size.width, MIN_SLIDER_WIDTH);
-    [self setFrame:rect display:YES];
-
-
-
 }
 
 - (CVView *)contentView {
@@ -888,20 +902,15 @@ void cv::setWindowTitle(const String& winname, const String& title)
 - (id)init {
     //cout << "CVView init" << endl;
     [super init];
-    image = [[NSImage alloc] init];
     return self;
 }
 
 - (void)setImageData:(CvArr *)arr {
     //cout << "setImageData" << endl;
     NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init];
-    CvMat *arrMat, *cvimage, stub;
+    CvMat *arrMat, dst, stub;
 
     arrMat = cvGetMat(arr, &stub);
-
-    cvimage = cvCreateMat(arrMat->rows, arrMat->cols, CV_8UC3);
-    cvConvertImage(arrMat, cvimage, CV_CVTIMG_SWAP_RB);
-
     /*CGColorSpaceRef colorspace = NULL;
     CGDataProviderRef provider = NULL;
     int width = cvimage->width;
@@ -922,42 +931,77 @@ void cv::setWindowTitle(const String& winname, const String& title)
     }*/
 
     NSBitmapImageRep *bitmap = [[NSBitmapImageRep alloc] initWithBitmapDataPlanes:NULL
-                pixelsWide:cvimage->width
+                pixelsWide:arrMat->cols
-                pixelsHigh:cvimage->height
+                pixelsHigh:arrMat->rows
                 bitsPerSample:8
                 samplesPerPixel:3
                 hasAlpha:NO
                 isPlanar:NO
                 colorSpaceName:NSDeviceRGBColorSpace
-                bytesPerRow:(cvimage->width * 4)
+                bitmapFormat: kCGImageAlphaNone
-                bitsPerPixel:32];
+                bytesPerRow:((arrMat->cols * 3 + 3) & -4)
+                bitsPerPixel:24];
 
-    int	pixelCount = cvimage->width * cvimage->height;
+    if (bitmap) {
-    unsigned char *src = cvimage->data.ptr;
+        cvInitMatHeader(&dst, arrMat->rows, arrMat->cols, CV_8UC3, [bitmap bitmapData], [bitmap bytesPerRow]);
-    unsigned char *dst = [bitmap bitmapData];
+        cvConvertImage(arrMat, &dst, CV_CVTIMG_SWAP_RB);
-
+    }
-    for( int i = 0; i < pixelCount; i++ )
+    else {
-    {
+        // It's not guaranteed to like the bitsPerPixel:24, but this is a lot slower so we'd rather not do it
-        dst[i * 4 + 0] = src[i * 3 + 0];
+        bitmap = [[NSBitmapImageRep alloc] initWithBitmapDataPlanes:NULL
-        dst[i * 4 + 1] = src[i * 3 + 1];
+            pixelsWide:arrMat->cols
-        dst[i * 4 + 2] = src[i * 3 + 2];
+            pixelsHigh:arrMat->rows
+            bitsPerSample:8
+            samplesPerPixel:3
+            hasAlpha:NO
+            isPlanar:NO
+            colorSpaceName:NSDeviceRGBColorSpace
+            bytesPerRow:(arrMat->cols * 4)
+            bitsPerPixel:32];
+        uint8_t *data = [bitmap bitmapData];
+        cvInitMatHeader(&dst, arrMat->rows, arrMat->cols, CV_8UC3, data, (arrMat->cols * 3));
+        cvConvertImage(arrMat, &dst, CV_CVTIMG_SWAP_RB);
+        for (int i = (arrMat->rows * arrMat->cols) - 1; i >= 0; i--) {
+            memmove(data + i * 4, data + i * 3, 3);
+            data[i * 4 + 3] = 0;
+        }
     }
 
-    if( image )
+    if( image ) {
         [image release];
+    }
 
     image = [[NSImage alloc] init];
     [image addRepresentation:bitmap];
     [bitmap release];
 
+    // This isn't supported on older versions of macOS
+    // The performance issues this solves are mainly on newer versions of macOS, so that's fine
+    if( floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_5 ) {
+        if (![self imageView]) {
+            [self setImageView:[[NSView alloc] init]];
+            [[self imageView] setWantsLayer:true];
+            [self addSubview:[self imageView]];
+        }
+
+        [[[self imageView] layer] setContents:image];
+
+        NSRect imageViewFrame = [self frame];
+        imageViewFrame.size.height -= [self sliderHeight];
+        NSRect constrainedFrame = { imageViewFrame.origin, constrainAspectRatio(imageViewFrame.size, [image size]) };
+        [[self imageView] setFrame:constrainedFrame];
+    }
+    else {
+        NSRect redisplayRect = [self frame];
+        redisplayRect.size.height -= [self sliderHeight];
+        [self setNeedsDisplayInRect:redisplayRect];
+    }
+
     /*CGColorSpaceRelease(colorspace);
     CGDataProviderRelease(provider);
     CGImageRelease(imageRef);*/
-    cvReleaseMat(&cvimage);
+
     [localpool drain];
-
-    [self setNeedsDisplay:YES];
-
 }
 
 - (void)setFrameSize:(NSSize)size {
@@ -970,41 +1014,49 @@ void cv::setWindowTitle(const String& winname, const String& title)
     CVWindow *cvwindow = (CVWindow *)[self window];
     if ([cvwindow respondsToSelector:@selector(sliders)]) {
         for(NSString *key in [cvwindow sliders]) {
-            NSSlider *slider = [[cvwindow sliders] valueForKey:key];
+            CVSlider *slider = [[cvwindow sliders] valueForKey:key];
             NSRect r = [slider frame];
             r.origin.y = height - r.size.height;
             r.size.width = [[cvwindow contentView] frame].size.width;
+
+            CGRect sliderRect = slider.slider.frame;
+            CGFloat targetWidth = r.size.width - (sliderRect.origin.x + 10);
+            sliderRect.size.width = targetWidth < 0 ? 0 : targetWidth;
+            slider.slider.frame = sliderRect;
+
             [slider setFrame:r];
             height -= r.size.height;
         }
     }
+    NSRect frame = self.frame;
+    if (frame.size.height < self.sliderHeight) {
+        frame.size.height = self.sliderHeight;
+        self.frame = frame;
+    }
+    if ([self imageView]) {
+        NSRect imageViewFrame = frame;
+        imageViewFrame.size.height -= [self sliderHeight];
+        NSRect constrainedFrame = { imageViewFrame.origin, constrainAspectRatio(imageViewFrame.size, [image size]) };
+        [[self imageView] setFrame:constrainedFrame];
+    }
     [localpool drain];
 }
 
 - (void)drawRect:(NSRect)rect {
     //cout << "drawRect" << endl;
     [super drawRect:rect];
+    // If imageView exists, all drawing will be done by it and nothing needs to happen here
+    if ([self image] && ![self imageView]) {
+        NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init];
 
-    NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init];
+        if(image != nil) {
-    CVWindow *cvwindow = (CVWindow *)[self window];
+            [image drawInRect: [self frame]
-    int height = 0;
+                     fromRect: NSZeroRect
-    if ([cvwindow respondsToSelector:@selector(sliders)]) {
+                    operation: NSCompositeSourceOver
-        for(NSString *key in [cvwindow sliders]) {
+                     fraction: 1.0];
-            height += [[[cvwindow sliders] valueForKey:key] frame].size.height;
         }
+        [localpool release];
     }
-
-
-    NSRect imageRect = {{0,0}, {[image size].width, [image size].height}};
-
-    if(image != nil) {
-        [image drawInRect: imageRect
-                              fromRect: NSZeroRect
-                             operation: NSCompositeSourceOver
-                              fraction: 1.0];
-    }
-    [localpool release];
-
 }
 
 @end