vo_opengl: add alpha output

Allows playing video with alpha information on X11, as long as the video
contains alpha and the window manager does compositing. See vo.rst.

Whether a window can be transparent is decided by the choice of the X
Visual used for window creation. Unfortunately, there's no direct way to
request such a Visual through the GLX or the X API, and use of the
XRender extension is required to find out whether a Visual implies a
framebuffer with alpha used by XRender (see for example [1]). Instead of
depending on the XRender wrapper library (which would require annoying
configure checks, even though XRender is virtually always supported),
use a simple heuristics to find out whether a Visual has alpha. Since
getting it wrong just means an optional feature will not work as
expected, we consider this ok.

[1] http://stackoverflow.com/questions/4052940/how-to-make-an-opengl-
rendering-context-with-transparent-background/9215724#9215724

DOCS/man/en/vo.rst

@@ -430,6 +430,14 @@ opengl
         dimension. Default is 128x256x64.
         Sizes must be a power of two, and 256 at most.
 
+    alpha
+        Try to create a framebuffer with alpha component. This only makes sense
+        if the video contains alpha information (which is extremely rare). May
+        not be supported on all platforms. If alpha framebuffers are
+        unavailable, it silently falls back to a normal framebuffer. Note
+        that when using FBO indirections (such as with ``opengl-hq``), a FBO
+        format with alpha must be specified with the ``fbo-format`` option.
+
 opengl-hq
     Same as ``opengl``, but with default settings for high quality rendering.
 

video/out/gl_video.c

@@ -253,6 +253,7 @@ const struct m_sub_options gl_video_conf = {
                     {"rgb16f", GL_RGB16F},
                     {"rgb32f", GL_RGB32F})),
         OPT_INTRANGE("dither-depth", dither_depth, 0, -1, 16),
+        OPT_FLAG("alpha", enable_alpha, 0),
         {0}
     },
     .size = sizeof(struct gl_video_opts),
@@ -668,6 +669,9 @@ static void compile_shaders(struct gl_video *p)
     char *header = talloc_asprintf(tmp, "#version %d\n%s", gl->glsl_version,
                                    shader_prelude);
 
+    // Need to pass alpha through the whole chain. (Not needed for OSD shaders.)
+    shader_def_opt(&header, "USE_ALPHA", p->opts.enable_alpha);
+
     char *header_osd = talloc_strdup(tmp, header);
     shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV", p->opts.srgb &&
                                                       !p->use_lut_3d);
@@ -701,6 +705,8 @@ static void compile_shaders(struct gl_video *p)
     shader_def_opt(&header_conv, "USE_YGRAY", p->is_yuv && p->plane_count == 1);
     shader_def_opt(&header_conv, "USE_COLORMATRIX", p->is_yuv);
     shader_def_opt(&header_conv, "USE_LINEAR_CONV", convert_input_to_linear);
+    if (p->opts.enable_alpha && p->plane_count == 4)
+        shader_def(&header_conv, "USE_ALPHA_PLANE", "3");
 
     shader_def_opt(&header_final, "USE_LINEAR_CONV_INV", p->use_lut_3d);
     shader_def_opt(&header_final, "USE_GAMMA_POW", p->opts.gamma);
@@ -1550,7 +1556,7 @@ static int init_gl(struct gl_video *p)
 
     gl->BindBuffer(GL_ARRAY_BUFFER, 0);
 
-    gl->ClearColor(0.0f, 0.0f, 0.0f, 0.0f);
+    gl->ClearColor(0.0f, 0.0f, 0.0f, 1.0f);
 
     debug_check_gl(p, "after init_gl");
 
@@ -1642,6 +1648,10 @@ static bool init_format(int fmt, struct gl_video *init)
     found: ;
     }
 
+    // Stuff like IMGFMT_420AP10. Untested, most likely insane.
+    if (desc.num_planes == 4 && (init->plane_bits % 8) != 0)
+        return false;
+
     init->is_yuv = desc.flags & MP_IMGFLAG_YUV;
     init->is_linear_rgb = false;
     init->plane_count = desc.num_planes;

video/out/gl_video.h

@@ -40,6 +40,7 @@ struct gl_video_opts {
     int dither_depth;
     int fbo_format;
     int stereo_mode;
+    int enable_alpha;
 };
 
 extern const struct m_sub_options gl_video_conf;

video/out/gl_video_shaders.glsl

@@ -112,8 +112,8 @@ void main() {
 }
 
 #!section frag_video
-uniform sampler2D textures[3];
+uniform sampler2D textures[4];
-uniform vec2 textures_size[3];
+uniform vec2 textures_size[4];
 uniform sampler1D lut_c_1d;
 uniform sampler1D lut_l_1d;
 uniform sampler2D lut_c_2d;
@@ -323,11 +323,19 @@ void main() {
     vec3 color = vec3(SAMPLE_L(textures[0], textures_size[0], texcoord).r,
                       SAMPLE_C(textures[1], textures_size[1], texcoord).r,
                       SAMPLE_C(textures[2], textures_size[2], texcoord).r);
+    float alpha = 1.0;
 #elif USE_CONV == CONV_NV12
     vec3 color = vec3(SAMPLE_L(textures[0], textures_size[0], texcoord).r,
                       SAMPLE_C(textures[1], textures_size[1], texcoord).rg);
+    float alpha = 1.0;
 #else
-    vec3 color = SAMPLE_L(textures[0], textures_size[0], texcoord).rgb;
+    vec4 acolor = SAMPLE_L(textures[0], textures_size[0], texcoord);
+    vec3 color = acolor.rgb;
+    float alpha = acolor.a;
+#endif
+#ifdef USE_ALPHA_PLANE
+    alpha = SAMPLE_L(textures[USE_ALPHA_PLANE],
+                     textures_size[USE_ALPHA_PLANE], texcoord).r;
 #endif
 #ifdef USE_GBRP
     color.gbr = color;
@@ -364,5 +372,9 @@ void main() {
     float dither_value = texture(dither, gl_FragCoord.xy / dither_size).r;
     color = floor(color * dither_multiply + dither_value ) / dither_quantization;
 #endif
-    out_color = vec4(color, 1);
+#ifdef USE_ALPHA
+    out_color = vec4(color, alpha);
+#else
+    out_color = vec4(color, 1.0);
+#endif
 }

video/out/gl_x11.c

@@ -149,7 +149,7 @@ static bool create_context_x11_gl3(struct MPGLContext *ctx, bool debug)
 //  http://www.opengl.org/wiki/Tutorial:_OpenGL_3.0_Context_Creation_(GLX)
 // but also uses some of the old code.
 
-static GLXFBConfig select_fb_config(struct vo *vo, const int *attribs)
+static GLXFBConfig select_fb_config(struct vo *vo, const int *attribs, int flags)
 {
     int fbcount;
     GLXFBConfig *fbc = glXChooseFBConfig(vo->x11->display, vo->x11->screen,
@@ -160,11 +160,37 @@ static GLXFBConfig select_fb_config(struct vo *vo, const int *attribs)
     // The list in fbc is sorted (so that the first element is the best).
     GLXFBConfig fbconfig = fbc[0];
 
+    if (flags & VOFLAG_ALPHA) {
+        for (int n = 0; n < fbcount; n++) {
+            XVisualInfo *v = glXGetVisualFromFBConfig(vo->x11->display, fbc[n]);
+            // This is a heuristic at best. Note that normal 8 bit Visuals use
+            // a depth of 24, even if the pixels are padded to 32 bit. If the
+            // depth is higher than 24, the remaining bits must be alpha.
+            // Note: vinfo->bits_per_rgb appears to be useless (is always 8).
+            unsigned long mask = v->depth == 32 ?
+                (unsigned long)-1 : (1 << (unsigned long)v->depth) - 1;
+            if (mask & ~(v->red_mask | v->green_mask | v->blue_mask)) {
+                fbconfig = fbc[n];
+                break;
+            }
+        }
+    }
+
     XFree(fbc);
 
     return fbconfig;
 }
 
+static void set_glx_attrib(int *attribs, int name, int value)
+{
+    for (int n = 0; attribs[n * 2 + 0] != None; n++) {
+        if (attribs[n * 2 + 0] == name) {
+            attribs[n * 2 + 1] = value;
+            break;
+        }
+    }
+}
+
 static bool config_window_x11(struct MPGLContext *ctx, uint32_t d_width,
                               uint32_t d_height, uint32_t flags)
 {
@@ -189,28 +215,37 @@ static bool config_window_x11(struct MPGLContext *ctx, uint32_t d_width,
         return false;
     }
 
-    const int glx_attribs_stereo_value_idx = 1; // index of GLX_STEREO + 1
     int glx_attribs[] = {
         GLX_STEREO, False,
         GLX_X_RENDERABLE, True,
         GLX_RED_SIZE, 1,
         GLX_GREEN_SIZE, 1,
         GLX_BLUE_SIZE, 1,
+        GLX_ALPHA_SIZE, 0,
         GLX_DOUBLEBUFFER, True,
         None
     };
     GLXFBConfig fbc = NULL;
+    if (flags & VOFLAG_ALPHA) {
+        set_glx_attrib(glx_attribs, GLX_ALPHA_SIZE, 1);
+        fbc = select_fb_config(vo, glx_attribs, flags);
+        if (!fbc) {
+            set_glx_attrib(glx_attribs, GLX_ALPHA_SIZE, 0);
+            flags &= ~VOFLAG_ALPHA;
+        }
+    }
     if (flags & VOFLAG_STEREO) {
-        glx_attribs[glx_attribs_stereo_value_idx] = True;
+        set_glx_attrib(glx_attribs, GLX_STEREO, True);
-        fbc = select_fb_config(vo, glx_attribs);
+        fbc = select_fb_config(vo, glx_attribs, flags);
         if (!fbc) {
             mp_msg(MSGT_VO, MSGL_ERR, "[gl] Could not find a stereo visual,"
                    " 3D will probably not work!\n");
-            glx_attribs[glx_attribs_stereo_value_idx] = False;
+            set_glx_attrib(glx_attribs, GLX_STEREO, False);
+            flags &= ~VOFLAG_STEREO;
         }
     }
     if (!fbc)
-        fbc = select_fb_config(vo, glx_attribs);
+        fbc = select_fb_config(vo, glx_attribs, flags);
     if (!fbc) {
         mp_msg(MSGT_VO, MSGL_ERR, "[gl] no GLX support present\n");
         return false;

video/out/vo.h

@@ -120,6 +120,7 @@ typedef struct {
 #define VOFLAG_HIDDEN		0x10  //< Use to create a hidden window
 #define VOFLAG_STEREO		0x20  //< Use to create a stereo-capable window
 #define VOFLAG_GL_DEBUG         0x40  // Hint to request debug OpenGL context
+#define VOFLAG_ALPHA            0x80  // Hint to request alpha framebuffer
 
 typedef struct vo_info_s
 {

video/out/vo_opengl.c

@@ -133,6 +133,9 @@ static bool config_window(struct gl_priv *p, uint32_t d_width,
     if (p->renderer_opts->stereo_mode == GL_3D_QUADBUFFER)
         flags |= VOFLAG_STEREO;
 
+    if (p->renderer_opts->enable_alpha)
+        flags |= VOFLAG_ALPHA;
+
     if (p->use_gl_debug)
         flags |= VOFLAG_GL_DEBUG;