rpms/gtk-vnc
5
0
Fork 0
Code Issues Pull Requests Releases Activity

Fix endianess conversion, protocol negotiation and makecontext() crash

Browse Source
This commit is contained in:
Daniel P. Berrange 2008-04-03 14:42:23 +00:00
parent 181819c551
commit 9c89670006
4 changed files with 431 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

322
gtk-vnc-0.3.4-endianess.patch Normal file
View File

@ -0,0 +1,322 @@
diff -rup gtk-vnc-0.3.4.orig/src/blt.h gtk-vnc-0.3.4.new/src/blt.h
--- gtk-vnc-0.3.4.orig/src/blt.h 2008-02-22 09:14:48.000000000 -0500
+++ gtk-vnc-0.3.4.new/src/blt.h 2008-04-03 10:15:01.000000000 -0400
@@ -16,8 +16,9 @@
#define RGB24_BLIT SPLICE(gvnc_rgb24_blt_, SUFFIX())
#define TIGHT_COMPUTE_PREDICTED SPLICE(gvnc_tight_compute_predicted_, SUFFIX())
#define TIGHT_SUM_PIXEL SPLICE(gvnc_tight_sum_pixel_, SUFFIX())
-#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, DST)(gvnc, pixel)
-#define COMPONENT(color, pixel) ((SWAP(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
+#define SWAP_RFB(gvnc, pixel) SPLICE(gvnc_swap_rfb_, SRC)(gvnc, pixel)
+#define SWAP_IMG(gvnc, pixel) SPLICE(gvnc_swap_img_, DST)(gvnc, pixel)
+#define COMPONENT(color, pixel) ((SWAP_RFB(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
static void FAST_FILL(struct gvnc *gvnc, src_pixel_t *sp,
int x, int y, int width, int height)
@@ -41,18 +42,18 @@ static void FAST_FILL(struct gvnc *gvnc,
}
}
-static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t *sp)
+static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t sp)
{
- *dp = SWAP(gvnc, ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
- | ((*sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
- | ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls);
+ *dp = SWAP_IMG(gvnc, ((sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
+ | ((sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
+ | ((sp >> gvnc->brs) & gvnc->bm) << gvnc->bls);
}
static void SET_PIXEL_AT(struct gvnc *gvnc, int x, int y, src_pixel_t *sp)
{
dst_pixel_t *dp = (dst_pixel_t *)gvnc_get_local(gvnc, x, y);
- SET_PIXEL(gvnc, dp, sp);
+ SET_PIXEL(gvnc, dp, SWAP_RFB(gvnc, *sp));
}
static void FILL(struct gvnc *gvnc, src_pixel_t *sp,
@@ -66,7 +67,7 @@ static void FILL(struct gvnc *gvnc, src_
int j;
for (j = 0; j < width; j++) {
- SET_PIXEL(gvnc, dp, sp);
+ SET_PIXEL(gvnc, dp, SWAP_RFB(gvnc, *sp));
dp++;
}
dst += gvnc->local.linesize;
@@ -88,7 +89,7 @@ static void BLIT(struct gvnc *gvnc, uint
int j;
for (j = 0; j < w; j++) {
- SET_PIXEL(gvnc, dp, sp);
+ SET_PIXEL(gvnc, dp, SWAP_RFB(gvnc, *sp));
dp++;
sp++;
}
@@ -171,11 +172,36 @@ static void RICH_CURSOR_BLIT(struct gvnc
uint32_t *dst = (uint32_t *)pixbuf;
uint8_t *src = image;
uint8_t *alpha = mask;
- int rs, gs, bs;
+ int as, rs, gs, bs, n;
- rs = 24 - ((sizeof(src_pixel_t) * 8) - gvnc->fmt.red_shift);
- gs = 16 - (gvnc->fmt.red_shift - gvnc->fmt.green_shift);
- bs = 8 - (gvnc->fmt.green_shift - gvnc->fmt.blue_shift);
+ /*
+ * GdkPixbuf is always 32-bit RGB, so we can't use the precomputed
+ * left / right shift data from gvnc->{r,g,b}{r,l}s. The latter
+ * is set for the local display depth, which may be different
+ * to GdkPixbuf's fixed 32-bit RGBA
+ *
+ * This function isn't called often, so just re-compute them now
+ */
+
+#if G_BYTE_ORDER == G_BIG_ENDIAN
+ as = 0;
+ rs = 8;
+ gs = 16;
+ bs = 24;
+#else
+ as = 24;
+ rs = 16;
+ gs = 8;
+ bs = 0;
+#endif
+
+ /* Then this adjusts for remote having less bpp than 32 */
+ for (n = 255 ; n > gvnc->fmt.red_max ; n>>= 1)
+ rs++;
+ for (n = 255 ; n > gvnc->fmt.green_max ; n>>= 1)
+ gs++;
+ for (n = 255 ; n > gvnc->fmt.blue_max ; n>>= 1)
+ bs++;
for (y1 = 0; y1 < height; y1++) {
src_pixel_t *sp = (src_pixel_t *)src;
@@ -186,7 +212,7 @@ static void RICH_CURSOR_BLIT(struct gvnc
| (COMPONENT(blue, *sp) << bs);
if ((mp[x1 / 8] >> (7 - (x1 % 8))) & 1)
- *dst |= 0xFF000000;
+ *dst |= (0xFF << as);
dst++;
sp++;
@@ -209,6 +235,11 @@ static void RGB24_BLIT(struct gvnc *gvnc
uint8_t *sp = data;
for (i = 0; i < width; i++) {
+ /*
+ * We use gvnc->fmt.XXX_shift instead of usual gvnc->Xls
+ * because the source pixel component is a full 8 bits in
+ * size, and so doesn't need the adjusted shift
+ */
*dp = (((sp[0] * gvnc->fmt.red_max) / 255) << gvnc->fmt.red_shift) |
(((sp[1] * gvnc->fmt.green_max) / 255) << gvnc->fmt.green_shift) |
(((sp[2] * gvnc->fmt.blue_max) / 255) << gvnc->fmt.blue_shift);
@@ -242,7 +273,7 @@ static void TIGHT_COMPUTE_PREDICTED(stru
blue = MAX(blue, 0);
blue = MIN(blue, gvnc->fmt.blue_max);
- *ppixel = SWAP(gvnc,
+ *ppixel = SWAP_RFB(gvnc,
(red << gvnc->fmt.red_shift) |
(green << gvnc->fmt.green_shift) |
(blue << gvnc->fmt.blue_shift));
@@ -257,7 +288,7 @@ static void TIGHT_SUM_PIXEL(struct gvnc
green = COMPONENT(green, *lhs) + COMPONENT(green, *rhs);
blue = COMPONENT(blue, *lhs) + COMPONENT(blue, *rhs);
- *lhs = SWAP(gvnc,
+ *lhs = SWAP_RFB(gvnc,
((red & gvnc->fmt.red_max) << gvnc->fmt.red_shift) |
((green & gvnc->fmt.green_max) << gvnc->fmt.green_shift) |
((blue & gvnc->fmt.blue_max) << gvnc->fmt.blue_shift));
diff -rup gtk-vnc-0.3.4.orig/src/gvnc.c gtk-vnc-0.3.4.new/src/gvnc.c
--- gtk-vnc-0.3.4.orig/src/gvnc.c 2008-02-22 09:33:46.000000000 -0500
+++ gtk-vnc-0.3.4.new/src/gvnc.c 2008-04-03 10:16:21.000000000 -0400
@@ -826,14 +826,39 @@ gboolean gvnc_set_pixel_format(struct gv
gboolean gvnc_set_encodings(struct gvnc *gvnc, int n_encoding, int32_t *encoding)
{
uint8_t pad[1] = {0};
- int i;
+ int i, skip_zrle=0;
+
+ /*
+ * RealVNC server is broken for ZRLE in some pixel formats.
+ * Specifically if you have a format with either R, G or B
+ * components with a max value > 255, it still uses a CPIXEL
+ * of 3 bytes, even though the colour requirs 4 bytes. It
+ * thus messes up the colours of the server in a way we can't
+ * recover from on the client. Most VNC clients don't see this
+ * problem since they send a 'set pixel format' message instead
+ * of running with the server's default format.
+ *
+ * So we kill off ZRLE encoding for problematic pixel formats
+ */
+ for (i = 0; i < n_encoding; i++)
+ if (gvnc->fmt.depth == 32 &&
+ (gvnc->fmt.red_max > 255 ||
+ gvnc->fmt.blue_max > 255 ||
+ gvnc->fmt.green_max > 255) &&
+ encoding[i] == GVNC_ENCODING_ZRLE) {
+ GVNC_DEBUG("Dropping ZRLE encoding for broken pixel format\n");
+ skip_zrle++;
+ }
gvnc->has_ext_key_event = FALSE;
gvnc_write_u8(gvnc, 2);
gvnc_write(gvnc, pad, 1);
- gvnc_write_u16(gvnc, n_encoding);
- for (i = 0; i < n_encoding; i++)
+ gvnc_write_u16(gvnc, n_encoding - skip_zrle);
+ for (i = 0; i < n_encoding; i++) {
+ if (skip_zrle && encoding[i] == GVNC_ENCODING_ZRLE)
+ continue;
gvnc_write_s32(gvnc, encoding[i]);
+ }
gvnc_flush(gvnc);
return !gvnc_has_error(gvnc);
}
@@ -951,23 +976,52 @@ static inline uint8_t *gvnc_get_local(st
(x * gvnc->local.bpp);
}
-static uint8_t gvnc_swap_8(struct gvnc *gvnc G_GNUC_UNUSED, uint8_t pixel)
+static uint8_t gvnc_swap_img_8(struct gvnc *gvnc G_GNUC_UNUSED, uint8_t pixel)
{
return pixel;
}
-static uint16_t gvnc_swap_16(struct gvnc *gvnc, uint16_t pixel)
+static uint8_t gvnc_swap_rfb_8(struct gvnc *gvnc G_GNUC_UNUSED, uint8_t pixel)
+{
+ return pixel;
+}
+
+/* local host native format -> X server image format */
+static uint16_t gvnc_swap_img_16(struct gvnc *gvnc, uint16_t pixel)
+{
+ if (G_BYTE_ORDER != gvnc->local.byte_order)
+ return (((pixel >> 8) & 0xFF) << 0) |
+ (((pixel >> 0) & 0xFF) << 8);
+ else
+ return pixel;
+}
+
+/* VNC server RFB format -> local host native format */
+static uint16_t gvnc_swap_rfb_16(struct gvnc *gvnc, uint16_t pixel)
{
- if (gvnc->fmt.byte_order != gvnc->local.byte_order)
+ if (gvnc->fmt.byte_order != G_BYTE_ORDER)
return (((pixel >> 8) & 0xFF) << 0) |
(((pixel >> 0) & 0xFF) << 8);
else
return pixel;
}
-static uint32_t gvnc_swap_32(struct gvnc *gvnc, uint32_t pixel)
+/* local host native format -> X server image format */
+static uint32_t gvnc_swap_img_32(struct gvnc *gvnc, uint32_t pixel)
+{
+ if (G_BYTE_ORDER != gvnc->local.byte_order)
+ return (((pixel >> 24) & 0xFF) << 0) |
+ (((pixel >> 16) & 0xFF) << 8) |
+ (((pixel >> 8) & 0xFF) << 16) |
+ (((pixel >> 0) & 0xFF) << 24);
+ else
+ return pixel;
+}
+
+/* VNC server RFB format -> local host native format */
+static uint32_t gvnc_swap_rfb_32(struct gvnc *gvnc, uint32_t pixel)
{
- if (gvnc->fmt.byte_order != gvnc->local.byte_order)
+ if (gvnc->fmt.byte_order != G_BYTE_ORDER)
return (((pixel >> 24) & 0xFF) << 0) |
(((pixel >> 16) & 0xFF) << 8) |
(((pixel >> 8) & 0xFF) << 16) |
@@ -1206,10 +1260,26 @@ static void gvnc_read_cpixel(struct gvnc
memset(pixel, 0, bpp);
- if (bpp == 4 && gvnc->fmt.true_color_flag && gvnc->fmt.depth == 24) {
- bpp = 3;
- if (gvnc->fmt.byte_order == __BIG_ENDIAN)
- pixel += 1;
+ if (bpp == 4 && gvnc->fmt.true_color_flag) {
+ int fitsInMSB = ((gvnc->fmt.red_shift > 7) &&
+ (gvnc->fmt.green_shift > 7) &&
+ (gvnc->fmt.blue_shift > 7));
+ int fitsInLSB = (((gvnc->fmt.red_max << gvnc->fmt.red_shift) < (1 << 24)) &&
+ ((gvnc->fmt.green_max << gvnc->fmt.green_shift) < (1 << 24)) &&
+ ((gvnc->fmt.blue_max << gvnc->fmt.blue_shift) < (1 << 24)));
+
+ /*
+ * We need to analyse the shifts to see if they fit in 3 bytes,
+ * rather than looking at the declared 'depth' for the format
+ * because despite what the RFB spec says, this is what RealVNC
+ * server actually does in practice.
+ */
+ if (fitsInMSB || fitsInLSB) {
+ bpp = 3;
+ if (gvnc->fmt.depth == 24 &&
+ gvnc->fmt.byte_order == G_BIG_ENDIAN)
+ pixel++;
+ }
}
gvnc_read(gvnc, pixel, bpp);
@@ -2995,7 +3065,8 @@ gboolean gvnc_set_local(struct gvnc *gvn
fb->red_shift == gvnc->fmt.red_shift &&
fb->green_shift == gvnc->fmt.green_shift &&
fb->blue_shift == gvnc->fmt.blue_shift &&
- fb->byte_order == gvnc->fmt.byte_order)
+ fb->byte_order == G_BYTE_ORDER &&
+ gvnc->fmt.byte_order == G_BYTE_ORDER)
gvnc->perfect_match = TRUE;
else
gvnc->perfect_match = FALSE;
@@ -3024,17 +3095,6 @@ gboolean gvnc_set_local(struct gvnc *gvn
gvnc->bls = gvnc->local.blue_shift;
- /* This adjusts for server/client endianness mismatch */
- if (__BYTE_ORDER != gvnc->fmt.byte_order) {
- gvnc->rrs = gvnc->fmt.bits_per_pixel - gvnc->rrs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
- gvnc->grs = gvnc->fmt.bits_per_pixel - gvnc->grs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
- gvnc->brs = gvnc->fmt.bits_per_pixel - gvnc->brs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
-
- GVNC_DEBUG("Flipped shifts red: %3d, green: %3d, blue: %3d\n",
- gvnc->rrs, gvnc->grs, gvnc->brs);
- }
-
-
/* This adjusts for remote having more bpp than local */
for (n = gvnc->fmt.red_max; n > gvnc->local.red_mask ; n>>= 1)
gvnc->rrs++;
diff -rup gtk-vnc-0.3.4.orig/src/vncdisplay.c gtk-vnc-0.3.4.new/src/vncdisplay.c
--- gtk-vnc-0.3.4.orig/src/vncdisplay.c 2008-04-03 10:10:51.000000000 -0400
+++ gtk-vnc-0.3.4.new/src/vncdisplay.c 2008-04-03 10:13:53.000000000 -0400
@@ -750,9 +750,17 @@ static void setup_gdk_image(VncDisplay *
VncDisplayPrivate *priv = obj->priv;
GdkVisual *visual;
- visual = gdk_drawable_get_visual(GTK_WIDGET(obj)->window);
+ visual = gdk_screen_get_system_visual(gdk_screen_get_default());
priv->image = gdk_image_new(GDK_IMAGE_FASTEST, visual, width, height);
+ GVNC_DEBUG("Visual mask: %3d %3d %3d\n shift: %3d %3d %3d\n",
+ visual->red_mask,
+ visual->green_mask,
+ visual->blue_mask,
+ visual->red_shift,
+ visual->green_shift,
+ visual->blue_shift);
+
priv->fb.red_mask = visual->red_mask >> visual->red_shift;
priv->fb.green_mask = visual->green_mask >> visual->green_shift;
priv->fb.blue_mask = visual->blue_mask >> visual->blue_shift;

51
gtk-vnc-0.3.4-makecontext.patch Normal file
View File

@ -0,0 +1,51 @@
changeset: 199:d2f49ba72d0b
tag: tip
user: "Daniel P. Berrange <berrange@redhat.com>"
date: Wed Apr 02 17:07:24 2008 -0400
summary: Fix makecontext() for 64-bit args
diff -r b28dadb1f19b -r d2f49ba72d0b src/continuation.c
--- a/src/continuation.c Wed Apr 02 15:12:25 2008 -0400
+++ b/src/continuation.c Wed Apr 02 17:07:24 2008 -0400
@@ -10,8 +10,31 @@
#include "continuation.h"
+/*
+ * va_args to makecontext() must be type 'int', so passing
+ * the pointer we need may require several int args. This
+ * union is a quick hack to let us do that
+ */
+union cc_arg {
+ void *p;
+ int i[2];
+};
+
+static void continuation_trampoline(int i0, int i1)
+{
+ union cc_arg arg;
+ struct continuation *cc;
+ arg.i[0] = i0;
+ arg.i[1] = i1;
+ cc = arg.p;
+
+ cc->entry(cc);
+}
+
int cc_init(struct continuation *cc)
{
+ union cc_arg arg;
+ arg.p = cc;
if (getcontext(&cc->uc) == -1)
return -1;
@@ -20,7 +43,7 @@ int cc_init(struct continuation *cc)
cc->uc.uc_stack.ss_size = cc->stack_size;
cc->uc.uc_stack.ss_flags = 0;
- makecontext(&cc->uc, (void *)cc->entry, 1, cc);
+ makecontext(&cc->uc, (void *)continuation_trampoline, 2, arg.i[0], arg.i[1]);
return 0;
}

46
gtk-vnc-0.3.4-protocol-version.patch Normal file
View File

@ -0,0 +1,46 @@
diff -rup gtk-vnc-0.3.4.orig/src/gvnc.c gtk-vnc-0.3.4.new/src/gvnc.c
--- gtk-vnc-0.3.4.orig/src/gvnc.c 2008-04-03 10:16:21.000000000 -0400
+++ gtk-vnc-0.3.4.new/src/gvnc.c 2008-04-03 10:24:25.000000000 -0400
@@ -2775,6 +2775,13 @@ gboolean gvnc_is_initialized(struct gvnc
return FALSE;
}
+static gboolean gvnc_before_version (struct gvnc *gvnc, int major, int minor) {
+ return (gvnc->major < major) || (gvnc->major == major && gvnc->minor < minor);
+}
+static gboolean gvnc_after_version (struct gvnc *gvnc, int major, int minor) {
+ return !gvnc_before_version (gvnc, major, minor+1);
+}
+
gboolean gvnc_initialize(struct gvnc *gvnc, gboolean shared_flag)
{
int ret, i;
@@ -2790,18 +2797,17 @@ gboolean gvnc_initialize(struct gvnc *gv
if (ret != 2)
goto fail;
- if (gvnc->major != 3)
- goto fail;
- if (gvnc->minor != 3 &&
- gvnc->minor != 4 &&
- gvnc->minor != 5 &&
- gvnc->minor != 6 &&
- gvnc->minor != 7 &&
- gvnc->minor != 8)
- goto fail;
+ GVNC_DEBUG("Server version: %d.%d\n", gvnc->major, gvnc->minor);
- /* For UltraVNC ... */
- if (gvnc->minor > 3 && gvnc->minor < 7) gvnc->minor = 3;
+ if (gvnc_before_version(gvnc, 3, 3)) {
+ GVNC_DEBUG("Server version is not supported (%d.%d)\n", gvnc->major, gvnc->minor);
+ goto fail;
+ } else if (gvnc_before_version(gvnc, 3, 7)) {
+ gvnc->minor = 3;
+ } else if (gvnc_after_version(gvnc, 3, 8)) {
+ gvnc->major = 3;
+ gvnc->minor = 8;
+ }
snprintf(version, 12, "RFB %03d.%03d\n", gvnc->major, gvnc->minor);
gvnc_write(gvnc, version, 12);

13
gtk-vnc.spec
View File

@ -7,11 +7,14 @@
Summary: A GTK widget for VNC clients
Name: gtk-vnc
Version: 0.3.4
Release: 1%{?dist}
Release: 2%{?dist}
License: LGPLv2+
Group: Development/Libraries
Source: http://downloads.sourceforge.net/%{name}/%{name}-%{version}.tar.gz
Patch1: %{name}-%{version}-scaling-crash.patch
Patch2: %{name}-%{version}-makecontext.patch
Patch3: %{name}-%{version}-endianess.patch
Patch4: %{name}-%{version}-protocol-version.patch
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
URL: http://gtk-vnc.sf.net/
BuildRequires: gtk2-devel pygtk2-devel python-devel zlib-devel
@ -69,6 +72,9 @@ browsers.
%prep
%setup -q
%patch1 -p1
%patch2 -p1
%patch3 -p1
%patch4 -p1
%build
%if %{with_plugin}
@ -122,6 +128,11 @@ rm -fr %{buildroot}
%endif
%changelog
* Thu Apr 3 2008 Daniel P. Berrange <berrange@redhat.com> - 0.3.4-2.fc9
- Fixed endianness conversions
- Fix makecontext() args crash on x86_64
- Fix protocol version negotiation
* Thu Mar 6 2008 Daniel P. Berrange <berrange@redhat.com> - 0.3.4-1.fc9
- Update to 0.3.4 release
- Fix crash with OpenGL scaling code