Merge pull request #25822 from mqcmd196:gtk3-gl-support

Support OpenGL GTK3 New API #25822

Fixes #20001

GSoC2024 Project

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake

CMakeLists.txt

@@ -1468,11 +1468,14 @@ if(WITH_GTK OR HAVE_GTK)
   else()
     status("    GTK+:" "NO")
   endif()
+
   if(HAVE_GTK)
     status(  "      GThread :" HAVE_GTHREAD THEN "YES (ver ${GTHREAD_VERSION})" ELSE NO)
+    if(NOT HAVE_GTK3)
       status(  "    GtkGlExt:" HAVE_GTKGLEXT THEN "YES (ver ${GTKGLEXT_VERSION})" ELSE NO)
     endif()
   endif()
+endif()
 
 if(WITH_FRAMEBUFFER OR HAVE_FRAMEBUFFER)
   status("    Framebuffer UI:" HAVE_FRAMEBUFFER THEN YES ELSE NO)

cmake/OpenCVFindLibsGUI.cmake

@@ -63,7 +63,7 @@ endif()
 ocv_update(OpenGL_GL_PREFERENCE LEGACY)
 ocv_clear_vars(HAVE_OPENGL HAVE_QT_OPENGL)
 if(WITH_OPENGL)
-  if(WITH_WIN32UI OR (HAVE_QT AND QT_QTOPENGL_FOUND) OR HAVE_GTKGLEXT)
+  if(WITH_WIN32UI OR (HAVE_QT AND QT_QTOPENGL_FOUND) OR HAVE_GTK3 OR (HAVE_GTK AND NOT HAVE_GTK3 AND HAVE_GTKGLEXT))
     find_package (OpenGL QUIET)
     if(OPENGL_FOUND)
       set(HAVE_OPENGL TRUE)

modules/core/include/opencv2/core/opengl.hpp

@@ -57,7 +57,7 @@ This section describes OpenGL interoperability.
 
 To enable OpenGL support, configure OpenCV using CMake with WITH_OPENGL=ON . Currently OpenGL is
 supported only with WIN32, GTK and Qt backends on Windows and Linux (MacOS and Android are not
-supported). For GTK backend gtkglext-1.0 library is required.
+supported). For GTK-2.0 backend gtkglext-1.0 library is required.
 
 To use OpenGL functionality you should first create OpenGL context (window or frame buffer). You can
 do this with namedWindow function or with other OpenGL toolkit (GLUT, for example).

modules/highgui/CMakeLists.txt

@@ -218,6 +218,9 @@ if(TARGET ocv.3rdparty.gtk3 OR TARGET ocv.3rdparty.gtk2)
   )
     if(__gtk_dependency STREQUAL "ocv.3rdparty.gtk3")
       set(OPENCV_HIGHGUI_BUILTIN_BACKEND "GTK3")
+      if(OPENGL_LIBRARIES)
+        list(APPEND HIGHGUI_LIBRARIES "${OPENGL_LIBRARIES}")
+      endif()
     elseif(__gtk_dependency STREQUAL "ocv.3rdparty.gtk2")
       set(OPENCV_HIGHGUI_BUILTIN_BACKEND "GTK2")
     else()

modules/highgui/cmake/detect_gtk.cmake

@@ -26,7 +26,8 @@ if(WITH_GTK)
   else()
     ocv_add_external_target(gthread "${GTHREAD_INCLUDE_DIRS}" "${GTHREAD_LIBRARIES}" "HAVE_GTHREAD")
   endif()
-  if((WITH_OPENGL OR HAVE_OPENGL) AND HAVE_GTK2)
+  if((WITH_OPENGL OR HAVE_OPENGL) AND (HAVE_GTK2 OR HAVE_GTK3))
+    if(HAVE_GTK2)
       ocv_check_modules(GTKGLEXT gtkglext-1.0)
       if(HAVE_GTKGLEXT)
         # HACK for https://github.com/opencv/opencv/issues/20850
@@ -37,14 +38,15 @@ if(WITH_GTK)
             list(APPEND GTKGLEXT_INCLUDE_DIRS_EXISTS "${p}")
           endif()
         endforeach()
-      ocv_add_external_target(gtkglext "${GTKGLEXT_INCLUDE_DIRS_EXISTS}" "${GTKGLEXT_LIBRARIES}" "HAVE_GTKGLEXT")
+        ocv_add_external_target(gtkglext "${GTKGLEXT_INCLUDE_DIRS}" "${GTKGLEXT_LIBRARIES}" "HAVE_GTKGLEXT")
+      endif()
     endif()
   endif()
 elseif(HAVE_GTK)
   ocv_add_external_target(gtk "${GTK_INCLUDE_DIRS}" "${GTK_LIBRARIES}" "${GTK_DEFINES};HAVE_GTK")
 endif()
 
-if(WITH_OPENGL AND HAVE_GTKGLEXT)
+if(WITH_OPENGL)
   find_package(OpenGL QUIET)
   if(OPENGL_FOUND)
     set(HAVE_OPENGL TRUE)

modules/highgui/src/window_gtk.cpp

@@ -46,10 +46,7 @@
 
 #include <gtk/gtk.h>
 
-#if (GTK_MAJOR_VERSION == 3) && defined(HAVE_OPENGL)
+#if (GTK_MAJOR_VERSION == 2) && defined(HAVE_OPENGL) && !defined(HAVE_GTKGLEXT)
-  #undef HAVE_OPENGL  // no support with GTK3
-#endif
-#if defined(HAVE_OPENGL) && !defined(HAVE_GTKGLEXT)
   #undef HAVE_OPENGL  // gtkglext is required
 #endif
 
@@ -68,10 +65,14 @@
 #endif
 
 #ifdef HAVE_OPENGL
+  #ifdef GTK_VERSION3
+    #include <gtk/gtkglarea.h>
+  #else
     #include <gtk/gtkgl.h>
-    #include <GL/gl.h>
     #include <GL/glu.h>
   #endif
+  #include <GL/gl.h>
+#endif
 
 #include <opencv2/core/utils/logger.hpp>
 #include "opencv2/imgproc.hpp"
@@ -570,7 +571,7 @@ struct CvWindow : CvUIBase
         last_key(0), flags(0), status(0),
         on_mouse(NULL), on_mouse_param(NULL)
 #ifdef HAVE_OPENGL
-        ,useGl(false), glDrawCallback(NULL), glDrawData(NULL)
+        ,useGl(false), glDrawCallback(NULL), glDrawData(NULL), glArea(NULL)
 #endif
     {
         CV_LOG_INFO(NULL, "OpenCV/UI: creating GTK window: " << window_name);
@@ -597,6 +598,7 @@ struct CvWindow : CvUIBase
 
     CvOpenGlDrawCallback glDrawCallback;
     void* glDrawData;
+    GtkWidget* glArea;
 #endif
 };
 
@@ -640,7 +642,7 @@ CV_IMPL int cvInitSystem( int argc, char** argv )
 
         setlocale(LC_NUMERIC,"C");
 
-        #ifdef HAVE_OPENGL
+        #if defined(HAVE_OPENGL) && not defined(GTK_VERSION3) // GTK3+ uses GtkGLArea so no need to check for GtkGLExt
             if (!gtk_gl_init_check(&argc, &argv))
             {
                 hasError = true;
@@ -907,11 +909,42 @@ double cvGetOpenGlProp_GTK(const char* name)
 // OpenGL support
 
 #ifdef HAVE_OPENGL
-
 namespace
 {
+
+#ifdef GTK_VERSION3
+
+    void glRealizeCallback(GtkGLArea* area, gpointer user_data) {
+        CV_UNUSED(user_data);
+        gtk_gl_area_make_current(area);
+        if (gtk_gl_area_get_error(area) != NULL)
+            CV_Error(cv::Error::OpenGlApiCallError, "OpenGL context is not initialized");
+    }
+
+    gboolean glRenderCallback(GtkGLArea* area, GdkGLContext* context, gpointer user_data) {
+        CV_UNUSED(context);
+        CvWindow* window = (CvWindow*)user_data;
+        gtk_gl_area_make_current(area);
+        if (gtk_gl_area_get_error(area) != NULL) {
+            CV_Error(cv::Error::OpenGlApiCallError, "OpenGL context is not initialized");
+            return FALSE;
+        }
+        if(window->glDrawCallback) {
+            window->glDrawCallback(window->glDrawData);
+        }
+//        gtk_gl_area_queue_render(area);
+        return TRUE;
+    }
+
+#endif
+
     void createGlContext(CvWindow* window)
     {
+        #ifdef GTK_VERSION3
+        g_signal_connect(window->glArea, "realize", G_CALLBACK(glRealizeCallback), window);
+        g_signal_connect(window->glArea, "render", G_CALLBACK(glRenderCallback), window);
+        #else
+
         GdkGLConfig* glconfig;
 
         // Try double-buffered visual
@@ -923,11 +956,24 @@ namespace
         if (!gtk_widget_set_gl_capability(window->widget, glconfig, NULL, TRUE, GDK_GL_RGBA_TYPE))
             CV_Error( cv::Error::OpenGlApiCallError, "Can't Create A GL Device Context" );
 
+        #endif
+
         window->useGl = true;
     }
 
     void drawGl(CvWindow* window)
     {
+        #ifdef GTK_VERSION3
+
+        GtkGLArea* gtkGlArea = GTK_GL_AREA(window->glArea);
+        if (gtk_gl_area_get_error(gtkGlArea) != NULL)
+            CV_Error(cv::Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context");
+
+        if (window->glDrawCallback)
+            window->glDrawCallback(window->glDrawData);
+
+        #else
+
         GdkGLContext* glcontext = gtk_widget_get_gl_context(window->widget);
         GdkGLDrawable* gldrawable = gtk_widget_get_gl_drawable(window->widget);
 
@@ -947,6 +993,8 @@ namespace
             glFlush();
 
         gdk_gl_drawable_gl_end(gldrawable);
+
+        #endif
     }
 }
 
@@ -1041,12 +1089,27 @@ static std::shared_ptr<CvWindow> namedWindow_(const std::string& name, int flags
 
     window->frame = gtk_window_new( GTK_WINDOW_TOPLEVEL );
 
-    window->paned = gtk_vbox_new( FALSE, 0 );
     window->widget = cvImageWidgetNew( flags );
+
+#if defined(HAVE_OPENGL) && defined(GTK_VERSION3)
+    if (flags & cv::WINDOW_OPENGL) {
+        window->glArea = gtk_gl_area_new();
+        gtk_container_add(GTK_CONTAINER(window->frame), window->glArea);
+        gtk_widget_show(window->glArea);
+    } else {
+        window->paned = gtk_vbox_new( FALSE, 0 );
         gtk_box_pack_end( GTK_BOX(window->paned), window->widget, TRUE, TRUE, 0 );
         gtk_widget_show( window->widget );
         gtk_container_add( GTK_CONTAINER(window->frame), window->paned );
         gtk_widget_show( window->paned );
+    }
+#else
+    window->paned = gtk_vbox_new( FALSE, 0 );
+    gtk_box_pack_end( GTK_BOX(window->paned), window->widget, TRUE, TRUE, 0 );
+    gtk_widget_show( window->widget );
+    gtk_container_add( GTK_CONTAINER(window->frame), window->paned );
+    gtk_widget_show( window->paned );
+#endif
 
 #ifndef HAVE_OPENGL
     if (flags & cv::WINDOW_OPENGL)
@@ -1122,9 +1185,6 @@ static std::shared_ptr<CvWindow> namedWindow_(const std::string& name, int flags
 
 CV_IMPL void cvSetOpenGlContext(const char* name)
 {
-    GdkGLContext* glcontext;
-    GdkGLDrawable* gldrawable;
-
     CV_Assert(name && "NULL name string");
 
     CV_LOCK_MUTEX();
@@ -1136,11 +1196,24 @@ CV_IMPL void cvSetOpenGlContext(const char* name)
     if (!window->useGl)
         CV_Error( cv::Error::OpenGlNotSupported, "Window doesn't support OpenGL" );
 
+#ifdef GTK_VERSION3
+
+    if(gtk_gl_area_get_error(GTK_GL_AREA(window->glArea)) != NULL)
+        CV_Error( cv::Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context");
+
+#else
+
+    GdkGLContext* glcontext;
+    GdkGLDrawable* gldrawable;
+
     glcontext = gtk_widget_get_gl_context(window->widget);
     gldrawable = gtk_widget_get_gl_drawable(window->widget);
 
     if (!gdk_gl_drawable_make_current(gldrawable, glcontext))
         CV_Error( cv::Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context" );
+
+#endif
+
 }
 
 CV_IMPL void cvUpdateWindow(const char* name)
@@ -1154,9 +1227,22 @@ CV_IMPL void cvUpdateWindow(const char* name)
         return;
 
     // window does not refresh without this
+#ifdef GTK_VERSION3
+
+    if ( GTK_IS_GL_AREA(window->glArea) ){
+        gtk_gl_area_queue_render(GTK_GL_AREA(window->glArea));
+    } else {
         gtk_widget_queue_draw( GTK_WIDGET(window->widget));
     }
 
+#else
+
+    gtk_widget_queue_draw( GTK_WIDGET(window->widget) );
+
+#endif
+
+}
+
 CV_IMPL void cvSetOpenGlDrawCallback(const char* name, CvOpenGlDrawCallback callback, void* userdata)
 {
     CV_Assert(name && "NULL name string");

samples/opengl/CMakeLists.txt

@@ -6,6 +6,9 @@ if(UNIX)
   find_package(X11 QUIET)
 endif()
 
+find_package(PkgConfig QUIET)
+pkg_search_module(EPOXY QUIET epoxy)
+
 SET(OPENCV_OPENGL_SAMPLES_REQUIRED_DEPS
   opencv_core
   opencv_imgproc
@@ -21,6 +24,9 @@ if(BUILD_EXAMPLES AND OCV_DEPENDENCIES_FOUND)
   if(NOT X11_FOUND)
     ocv_list_filterout(all_samples "opengl_interop")
   endif()
+  if(NOT EPOXY_FOUND)
+    ocv_list_filterout(all_samples "opengl3_2")
+  endif()
   foreach(sample_filename ${all_samples})
     ocv_define_sample(tgt ${sample_filename} opengl)
     ocv_target_link_libraries(${tgt} PRIVATE "${OPENGL_LIBRARIES}" "${OPENCV_OPENGL_SAMPLES_REQUIRED_DEPS}")
@@ -28,6 +34,10 @@ if(BUILD_EXAMPLES AND OCV_DEPENDENCIES_FOUND)
       ocv_target_link_libraries(${tgt} PRIVATE ${X11_LIBRARIES})
       ocv_target_include_directories(${tgt} ${X11_INCLUDE_DIR})
     endif()
+    if(sample_filename STREQUAL "opengl3_2.cpp")
+      ocv_target_link_libraries(${tgt} PRIVATE ${EPOXY_LIBRARIES})
+      ocv_target_include_directories(${tgt} PRIVATE ${EPOXY_INCLUDE_DIRS})
+    endif()
   endforeach()
 endif()
 

samples/opengl/opengl3_2.cpp

@@ -0,0 +1,169 @@
+#include <iostream>
+
+#include <epoxy/gl.h>
+
+#ifdef _WIN32
+    #define WIN32_LEAN_AND_MEAN 1
+    #define NOMINMAX 1
+    #include <windows.h>
+#endif
+
+#if defined(__APPLE__)
+    #include <OpenGL/gl.h>
+    #include <OpenGL/glu.h>
+#else
+    #include <GL/gl.h>
+    #include <GL/glu.h>
+#endif
+
+#include "opencv2/core.hpp"
+#include "opencv2/core/opengl.hpp"
+#include "opencv2/core/cuda.hpp"
+#include "opencv2/highgui.hpp"
+
+using namespace std;
+using namespace cv;
+using namespace cv::cuda;
+
+const int win_width = 800;
+const int win_height = 640;
+
+struct DrawData
+{
+    GLuint vao, vbo, program, textureID;
+};
+
+static cv::Mat rot(float angle)
+{
+    cv::Mat R_y = (cv::Mat_<float>(4,4) <<
+        cos(angle), 0, sin(angle), 0,
+        0, 1, 0, 0,
+        -sin(angle), 0, cos(angle), 0,
+        0, 0, 0, 1);
+
+    return R_y;
+}
+
+static GLuint create_shader(const char* source, GLenum type) {
+    GLuint shader = glCreateShader(type);
+    glShaderSource(shader, 1, &source, NULL);
+    glCompileShader(shader);
+    return shader;
+}
+
+static void draw(void* userdata) {
+    DrawData* data = static_cast<DrawData*>(userdata);
+    static float angle = 0.0f;
+    angle += 1.f;
+
+    cv::Mat trans = rot(CV_PI * angle / 360.f);
+
+    glClearColor(0.0, 0.0, 0.0, 1.0);
+    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+
+    glUseProgram(data->program);
+    glUniformMatrix4fv(glGetUniformLocation(data->program, "transform"), 1, GL_FALSE, trans.ptr<float>());
+    glBindTexture(GL_TEXTURE_2D, data->textureID);
+    glBindVertexArray(data->vao);
+    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+    glBindVertexArray(0);
+}
+
+int main(int argc, char* argv[])
+{
+    string filename;
+    if (argc < 2)
+    {
+        cout << "Usage: " << argv[0] << " image" << endl;
+        filename = "baboon.jpg";
+    }
+    else
+        filename = argv[1];
+
+    Mat img = imread(samples::findFile(filename));
+    if (img.empty())
+    {
+        cerr << "Can't open image " << filename << endl;
+        return -1;
+    }
+    flip(img, img, 0);
+
+    namedWindow("OpenGL", WINDOW_OPENGL);
+    resizeWindow("OpenGL", win_width, win_height);
+
+    DrawData data;
+
+    glEnable(GL_DEPTH_TEST);
+    const char *vertex_shader_source =
+            "#version 330 core\n"
+            "layout (location = 0) in vec3 position;\n"
+            "layout (location = 1) in vec2 texCoord;\n"
+            "out vec2 TexCoord;\n"
+            "uniform mat4 transform;\n"
+            "void main() {\n"
+            "   gl_Position = transform * vec4(position, 1.0);\n"
+            "   TexCoord = texCoord;\n"
+            "}\n";
+    const char *fragment_shader_source =
+            "#version 330 core\n"
+            "in vec2 TexCoord;\n"
+            "out vec4 color;\n"
+            "uniform sampler2D ourTexture;\n"
+            "void main() {\n"
+            "   color = texture(ourTexture, TexCoord);\n"
+            "}\n";
+    data.program = glCreateProgram();
+    GLuint vertex_shader = create_shader(vertex_shader_source, GL_VERTEX_SHADER);
+    GLuint fragment_shader = create_shader(fragment_shader_source, GL_FRAGMENT_SHADER);
+    glAttachShader(data.program, vertex_shader);
+    glAttachShader(data.program, fragment_shader);
+    glLinkProgram(data.program);
+    glUseProgram(data.program);
+
+    GLfloat vertices[] = {
+            // Positions        // Texture Coords
+            1.0f,  1.0f, 0.0f,  1.0f, 1.0f,   // Top Right
+            1.0f, -1.0f, 0.0f,  1.0f, 0.0f,   // Bottom Right
+            -1.0f,  1.0f, 0.0f,  0.0f, 1.0f,   // Top Left
+            -1.0f, -1.0f, 0.0f,  0.0f, 0.0f    // Bottom Left
+    };
+
+    glGenVertexArrays(1, &data.vao);
+    glGenBuffers(1, &data.vbo);
+    glBindVertexArray(data.vao);
+    glBindBuffer(GL_ARRAY_BUFFER, data.vbo);
+    glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+
+    // Position attribute
+    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * sizeof(GLfloat), (GLvoid*)0);
+    glEnableVertexAttribArray(0);
+    // Texture Coord attribute
+    glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * sizeof(GLfloat), (GLvoid*)(3 * sizeof(GLfloat)));
+    glEnableVertexAttribArray(1);
+    glBindVertexArray(0); // Unbind VAO
+
+
+//        Image to texture
+    glGenTextures(1, &data.textureID);
+    glBindTexture(GL_TEXTURE_2D, data.textureID);
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, img.cols, img.rows, 0, GL_BGR, GL_UNSIGNED_BYTE, img.data);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glBindTexture(GL_TEXTURE_2D, 0);
+
+    setOpenGlDrawCallback("OpenGL", draw, &data);
+
+    for (;;)
+    {
+        updateWindow("OpenGL");
+        char key = (char)waitKey(40);
+        if (key == 27)
+            break;
+    }
+
+    setOpenGlDrawCallback("OpenGL", 0, 0);
+    destroyAllWindows();
+    return 0;
+}