ui: Add dpy_gfx_check_format() to check backend shared surface support

This allows VGA to decide whether to use a shared surface based on
whether the UI backend supports the format or not. Backends that
don't provide the new callback fallback to native 32 bpp which
is equivalent to what was supported before.
Signed-off-by: NBenjamin Herrenschmidt <benh@kernel.crashing.org>

[ kraxel: fix console check, allow only 32 bpp as fallback ]
Signed-off-by: NGerd Hoffmann <kraxel@redhat.com>

hw/display/vga.c

@@ -1437,6 +1437,7 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
     uint32_t v, addr1, addr;
     vga_draw_line_func *vga_draw_line = NULL;
     bool share_surface;
+    pixman_format_code_t format;
 #ifdef HOST_WORDS_BIGENDIAN
     bool byteswap = !s->big_endian_fb;
 #else
@@ -1481,8 +1482,19 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
 
     depth = s->get_bpp(s);
 
-    share_surface = (!s->force_shadow) &&
-            ( depth == 32 || (depth == 16 && !byteswap) );
+    /*
+     * Check whether we can share the surface with the backend
+     * or whether we need a shadow surface. We share native
+     * endian surfaces for 15bpp and above and byteswapped
+     * surfaces for 24bpp and above.
+     */
+    format = qemu_default_pixman_format(depth, !byteswap);
+    if (format) {
+        share_surface = dpy_gfx_check_format(s->con, format)
+            && !s->force_shadow;
+    } else {
+        share_surface = false;
+    }
     if (s->line_offset != s->last_line_offset ||
         disp_width != s->last_width ||
         height != s->last_height ||
@@ -1490,8 +1502,6 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         s->last_byteswap != byteswap ||
         share_surface != is_buffer_shared(surface)) {
         if (share_surface) {
-            pixman_format_code_t format =
-                qemu_default_pixman_format(depth, !byteswap);
             surface = qemu_create_displaysurface_from(disp_width,
                     height, format, s->line_offset,
                     s->vram_ptr + (s->start_addr * 4));

include/ui/console.h

@@ -161,6 +161,8 @@ typedef struct DisplayChangeListenerOps {
     void (*dpy_gfx_copy)(DisplayChangeListener *dcl,
                          int src_x, int src_y,
                          int dst_x, int dst_y, int w, int h);
+    bool (*dpy_gfx_check_format)(DisplayChangeListener *dcl,
+                                 pixman_format_code_t format);
 
     void (*dpy_text_cursor)(DisplayChangeListener *dcl,
                             int x, int y);
@@ -235,6 +237,8 @@ void dpy_gfx_update_dirty(QemuConsole *con,
                           MemoryRegion *address_space,
                           uint64_t base,
                           bool invalidate);
+bool dpy_gfx_check_format(QemuConsole *con,
+                          pixman_format_code_t format);
 
 static inline int surface_stride(DisplaySurface *s)
 {

ui/console.c

@@ -1439,6 +1439,31 @@ void dpy_gfx_replace_surface(QemuConsole *con,
     qemu_free_displaysurface(old_surface);
 }
 
+bool dpy_gfx_check_format(QemuConsole *con,
+                          pixman_format_code_t format)
+{
+    DisplayChangeListener *dcl;
+    DisplayState *s = con->ds;
+
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->con && dcl->con != con) {
+            /* dcl bound to another console -> skip */
+            continue;
+        }
+        if (dcl->ops->dpy_gfx_check_format) {
+            if (!dcl->ops->dpy_gfx_check_format(dcl, format)) {
+                return false;
+            }
+        } else {
+            /* default is to whitelist native 32 bpp only */
+            if (format != qemu_default_pixman_format(32, true)) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
 static void dpy_refresh(DisplayState *s)
 {
     DisplayChangeListener *dcl;