ui/vnc: fix vmware VGA incompatiblities

this fixes invalid rectangle updates observed after commit 12b316d4
with the vmware VGA driver. The issues occured because the server
and client surface update seems to be out of sync at some points
and the max width of the surface is not dividable by
VNC_DIRTY_BITS_PER_PIXEL (16).
Reported-by: NSerge Hallyn <serge.hallyn@ubuntu.com>
Signed-off-by: NPeter Lieven <pl@kamp.de>
Signed-off-by: NGerd Hoffmann <kraxel@redhat.com>

hw/display/vmware_vga.c

@@ -25,6 +25,7 @@
 #include "hw/loader.h"
 #include "trace.h"
 #include "ui/console.h"
+#include "ui/vnc.h"
 #include "hw/pci/pci.h"
 
 #undef VERBOSE
@@ -218,7 +219,7 @@ enum {
 
 /* These values can probably be changed arbitrarily.  */
 #define SVGA_SCRATCH_SIZE               0x8000
-#define SVGA_MAX_WIDTH                  2360
+#define SVGA_MAX_WIDTH                  ROUND_UP(2360, VNC_DIRTY_PIXELS_PER_BIT)
 #define SVGA_MAX_HEIGHT                 1770
 
 #ifdef VERBOSE

ui/vnc.c

@@ -888,7 +888,7 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
         VncDisplay *vd = vs->vd;
         VncJob *job;
         int y;
-        int height;
+        int height, width;
         int n = 0;
 
         if (vs->output.offset && !vs->audio_cap && !vs->force_update)
@@ -907,6 +907,7 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
         job = vnc_job_new(vs);
 
         height = MIN(pixman_image_get_height(vd->server), vs->client_height);
+        width = MIN(pixman_image_get_width(vd->server), vs->client_width);
 
         y = 0;
         for (;;) {
@@ -925,8 +926,11 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
                                     VNC_DIRTY_BPL(vs), x);
             bitmap_clear(vs->dirty[y], x, x2 - x);
             h = find_and_clear_dirty_height(vs, y, x, x2, height);
-            n += vnc_job_add_rect(job, x * VNC_DIRTY_PIXELS_PER_BIT, y,
-                                  (x2 - x) * VNC_DIRTY_PIXELS_PER_BIT, h);
+            x2 = MIN(x2, width / VNC_DIRTY_PIXELS_PER_BIT);
+            if (x2 > x) {
+                n += vnc_job_add_rect(job, x * VNC_DIRTY_PIXELS_PER_BIT, y,
+                                      (x2 - x) * VNC_DIRTY_PIXELS_PER_BIT, h);
+            }
         }
 
         vnc_job_push(job);