From eb8c7f16417bd033da75e0ba4edbe2821d1f417a Mon Sep 17 00:00:00 2001 From: Gregory Nutt Date: Thu, 21 Apr 2016 08:16:33 -0600 Subject: [PATCH] VNC: Correct pixel width in RRE encoding --- graphics/vnc/server/vnc_negotiate.c | 11 +- graphics/vnc/server/vnc_rre.c | 208 +++++++++++++++++++++------- graphics/vnc/server/vnc_server.h | 9 +- include/nuttx/video/rfb.h | 50 +++++++ 4 files changed, 219 insertions(+), 59 deletions(-) diff --git a/graphics/vnc/server/vnc_negotiate.c b/graphics/vnc/server/vnc_negotiate.c index bf3671bd1d..bb6aff34f4 100644 --- a/graphics/vnc/server/vnc_negotiate.c +++ b/graphics/vnc/server/vnc_negotiate.c @@ -384,12 +384,13 @@ int vnc_negotiate(FAR struct vnc_session_s *session) } else if (setformat->msgtype != RFB_SETPIXELFMT_MSG) { - gdbg("ERROR: Not a SetPixelFormat message: %d\n", (int)setformat->msgtype); + gdbg("ERROR: Not a SetPixelFormat message: %d\n", + (int)setformat->msgtype); return -EPROTO; } - /* Instantiate the client pixel format, verifying that the client request format - * is one that we can handle. + /* Instantiate the client pixel format, verifying that the client request + * format is one that we can handle. */ ret = vnc_client_pixelformat(session, &setformat->format); @@ -408,7 +409,7 @@ int vnc_negotiate(FAR struct vnc_session_s *session) encodings = (FAR struct rfb_setencodings_s *)session->inbuf; nrecvd = psock_recv(&session->connect, encodings, - CONFIG_VNCSERVER_INBUFFER_SIZE, 0); + CONFIG_VNCSERVER_INBUFFER_SIZE, 0); if (nrecvd < 0) { errcode = get_errno(); @@ -419,7 +420,7 @@ int vnc_negotiate(FAR struct vnc_session_s *session) if (nrecvd > 0 && encodings->msgtype == RFB_SETENCODINGS_MSG) { - DEBUGASSERT(nrecvd >= SIZEOF_RFB_SETENCODINGS_S(0)); + DEBUGASSERT(nrecvd >= SIZEOF_RFB_SETENCODINGS_S(0)); /* Pick out any mutually supported encodings */ diff --git a/graphics/vnc/server/vnc_rre.c b/graphics/vnc/server/vnc_rre.c index a271efa9d6..f012fae2a4 100644 --- a/graphics/vnc/server/vnc_rre.c +++ b/graphics/vnc/server/vnc_rre.c @@ -50,35 +50,119 @@ * Private Types ****************************************************************************/ -struct rre_data_s +struct rre_encode8_s { -#if RFB_BITSPERPIXEL == 16 - struct rfb_rrehdr16_s hdr; /* Header with background pixel */ - struct rfb_rrerect16_s srect; /* Sub-rectangle */ -#elif RFB_BITSPERPIXEL == 32 - struct rfb_rrehdr32_s hdr; /* Header with background pixel */ - struct rfb_rrerect32_s srect; /* Sub-rectangle */ -#endif + struct rfb_rrehdr8_s hdr; + struct rfb_rrerect8_s rect; }; -struct rre_rectangle_s +struct rre_encode16_s { - uint8_t xpos[2]; /* U16 X position */ - uint8_t ypos[2]; /* U16 Y position */ - uint8_t width[2]; /* U16 Width */ - uint8_t height[2]; /* U16 Height */ - uint8_t encoding[4]; /* S32 Encoding type = RFB_ENCODING_RRE */ - struct rre_data_s data; /* Pixel data, actual size varies */ + struct rfb_rrehdr16_s hdr; + struct rfb_rrerect16_s rect; }; -struct rre_framebufferupdate_s +struct rre_encode32_s { - uint8_t msgtype; /* U8 Message type = RFB_FBUPDATE_MSG*/ - uint8_t padding; - uint8_t nrect[2]; /* U16 Number of rectangles = 1*/ - struct rre_rectangle_s rect; /* RRE encoded rectangle */ + struct rfb_rrehdr32_s hdr; + struct rfb_rrerect32_s rect; }; +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: vnc_rreNN + * + * Description: + * Encode a single RRE sub-rectangle, NN=bits-per-pixel + * + * Input Parameters: + * session - An instance of the session structure. + * dest - The locate to save the RRE encoded data + * rect - Describes the rectangle in the local framebuffer. + * bgcolor - The local color of the pixel data + * + * Returned Value: + * The size of the framebuffer update message is returned.. + * + ****************************************************************************/ + +ssize_t vnc_rre8(FAR struct vnc_session_s *session, + FAR struct rre_encode8_s *dest, + FAR struct nxgl_rect_s *rect, + uint8_t bgcolor) +{ + nxgl_coord_t width; + nxgl_coord_t height; + + rfb_putbe32(dest->hdr.nsubrects, 1); + dest->hdr.pixel = bgcolor; + + dest->rect.pixel = bgcolor; + rfb_putbe16(dest->rect.xpos, rect->pt1.x); + rfb_putbe16(dest->rect.ypos, rect->pt1.y); + + width = rect->pt2.x - rect->pt1.x + 1; + height = rect->pt2.y - rect->pt1.y + 1; + + rfb_putbe16(dest->rect.width, width); + rfb_putbe16(dest->rect.height, height); + + return sizeof(struct rre_encode8_s); +} + +ssize_t vnc_rre16(FAR struct vnc_session_s *session, + FAR struct rre_encode16_s *dest, + FAR struct nxgl_rect_s *rect, + uint16_t bgcolor) +{ + nxgl_coord_t width; + nxgl_coord_t height; + + rfb_putbe32(dest->hdr.nsubrects, 1); + rfb_putbe16(dest->hdr.pixel, bgcolor); + + rfb_putbe16(dest->rect.pixel, bgcolor); + rfb_putbe16(dest->rect.xpos, rect->pt1.x); + rfb_putbe16(dest->rect.xpos, rect->pt1.x); + rfb_putbe16(dest->rect.ypos, rect->pt1.y); + + width = rect->pt2.x - rect->pt1.x + 1; + height = rect->pt2.y - rect->pt1.y + 1; + + rfb_putbe16(dest->rect.width, width); + rfb_putbe16(dest->rect.height, height); + + return sizeof(struct rre_encode16_s); +} + +ssize_t vnc_rre32(FAR struct vnc_session_s *session, + FAR struct rre_encode32_s *dest, + FAR struct nxgl_rect_s *rect, + uint32_t bgcolor) +{ + nxgl_coord_t width; + nxgl_coord_t height; + + rfb_putbe32(dest->hdr.nsubrects, 1); + rfb_putbe32(dest->hdr.pixel, bgcolor); + + rfb_putbe32(dest->rect.pixel, bgcolor); + rfb_putbe16(dest->rect.xpos, rect->pt1.x); + rfb_putbe16(dest->rect.xpos, rect->pt1.x); + rfb_putbe16(dest->rect.ypos, rect->pt1.y); + + width = rect->pt2.x - rect->pt1.x + 1; + height = rect->pt2.y - rect->pt1.y + 1; + + rfb_putbe16(dest->rect.width, width); + rfb_putbe16(dest->rect.height, height); + + return sizeof(struct rre_encode32_s); +} + /**************************************************************************** * Public Functions ****************************************************************************/ @@ -97,22 +181,21 @@ struct rre_framebufferupdate_s * * Returned Value: * Zero is returned if RRE coding was not performed (but not error was) - * encountered. Otherwise, eith the size of the framebuffer update - * message is returned on success or a negated errno value is returned on - * failure that indicates the the nature of the failure. A failure is - * only returned in cases of a network failure and unexpected internal - * failures. + * encountered. Otherwise, the size of the framebuffer update message + * is returned on success or a negated errno value is returned on failure + * that indicates the the nature of the failure. A failure is only + * returned in cases of a network failure and unexpected internal failures. * ****************************************************************************/ int vnc_rre(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect) { - FAR struct rre_framebufferupdate_s *rre; - FAR struct rre_rectangle_s *rrect; - FAR struct rre_data_s *rdata; + FAR struct rfb_framebufferupdate_s *rre; + FAR struct rfb_rectangle_s *rrect; lfb_color_t bgcolor; nxgl_coord_t width; nxgl_coord_t height; + size_t nbytes; ssize_t nsent; int ret; @@ -125,38 +208,66 @@ int vnc_rre(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect) ret = vnc_colors(session, rect, 1, &bgcolor); if (ret == 1) { - width = rect->pt2.x - rect->pt1.x + 1; - height = rect->pt2.y - rect->pt1.y + 1; + width = rect->pt2.x - rect->pt1.x + 1; + height = rect->pt2.y - rect->pt1.y + 1; /* Format the FrameBuffer Update with a single RRE encoded * rectangle. */ - rre = (FAR struct rre_framebufferupdate_s *)session->outbuf; + rre = (FAR struct rfb_framebufferupdate_s *)session->outbuf; rre->msgtype = RFB_FBUPDATE_MSG; rre->padding = 0; rfb_putbe16(rre->nrect, 1); - rrect = (FAR struct rre_rectangle_s *)&rre->rect; + rrect = (FAR struct rfb_rectangle_s *)&rre->rect; rfb_putbe16(rrect->xpos, rect->pt1.x); rfb_putbe16(rrect->ypos, rect->pt1.y); rfb_putbe16(rrect->width, width); rfb_putbe16(rrect->height, height); rfb_putbe32(rrect->encoding, RFB_ENCODING_RRE); - rdata = (FAR struct rre_data_s *)&rrect->data; - rfb_putbe32(rdata->hdr.nsubrects, 1); -#if RFB_BITSPERPIXEL == 16 - rfb_putbe16(rdata->hdr.pixel, bgcolor); - rfb_putbe16(rdata->srect.pixel, bgcolor); -#elif RFB_BITSPERPIXEL == 32 - rfb_putbe32(rdata->hdr.pixel, bgcolor); - rfb_putbe32(rdata->srect.pixel, bgcolor); -#endif - rfb_putbe16(rdata->srect.xpos, rect->pt1.x); - rfb_putbe16(rdata->srect.ypos, rect->pt1.y); - rfb_putbe16(rdata->srect.width, width); - rfb_putbe16(rdata->srect.height, height); + /* The sub-rectangle encoding depends of the remote pixel width */ + + nbytes = SIZEOF_RFB_FRAMEBUFFERUPDATE_S(SIZEOF_RFB_RECTANGE_S(0)); + + switch (session->colorfmt) + { + case FB_FMT_RGB8_222: + nbytes += vnc_rre8(session, + (FAR struct rre_encode8_s *)rrect->data, + rect, vnc_convert_rgb8_222(bgcolor)); + break; + + case FB_FMT_RGB8_332: + nbytes += vnc_rre8(session, + (FAR struct rre_encode8_s *)rrect->data, + rect, vnc_convert_rgb8_332(bgcolor)); + break; + + case FB_FMT_RGB16_555: + nbytes += vnc_rre16(session, + (FAR struct rre_encode16_s *)rrect->data, + rect, vnc_convert_rgb16_555(bgcolor)); + break; + + case FB_FMT_RGB16_565: + nbytes += vnc_rre16(session, + (FAR struct rre_encode16_s *)rrect->data, + rect, vnc_convert_rgb16_565(bgcolor)); + break; + + case FB_FMT_RGB32: + nbytes += vnc_rre32(session, + (FAR struct rre_encode32_s *)rrect->data, + rect, vnc_convert_rgb32_888(bgcolor)); + break; + + default: + gdbg("ERROR: Unrecognized color format: %d\n", + session->colorfmt); + return -EINVAL; + } /* At the very last most, make certain that the supported encoding * has not changed asynchronously. @@ -169,8 +280,7 @@ int vnc_rre(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect) * and there are a limited number of IOBs available. */ - nsent = psock_send(&session->connect, rre, - sizeof(struct rre_framebufferupdate_s), 0); + nsent = psock_send(&session->connect, rre, nbytes, 0); if (nsent < 0) { int errcode = get_errno(); @@ -180,8 +290,8 @@ int vnc_rre(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect) return -errcode; } - DEBUGASSERT(nsent == sizeof(struct rre_framebufferupdate_s)); - return sizeof(struct rre_framebufferupdate_s); + DEBUGASSERT(nsent == nbytes); + return nbytes; } return -EINVAL; diff --git a/graphics/vnc/server/vnc_server.h b/graphics/vnc/server/vnc_server.h index 9b7b059615..86e5d46e2e 100644 --- a/graphics/vnc/server/vnc_server.h +++ b/graphics/vnc/server/vnc_server.h @@ -455,11 +455,10 @@ int vnc_receiver(FAR struct vnc_session_s *session); * * Returned Value: * Zero is returned if RRE coding was not performed (but not error was) - * encountered. Otherwise, eith the size of the framebuffer update - * message is returned on success or a negated errno value is returned on - * failure that indicates the the nature of the failure. A failure is - * only returned in cases of a network failure and unexpected internal - * failures. + * encountered. Otherwise, the size of the framebuffer update message + * is returned on success or a negated errno value is returned on failure + * that indicates the the nature of the failure. A failure is only + * returned in cases of a network failure and unexpected internal failures. * ****************************************************************************/ diff --git a/include/nuttx/video/rfb.h b/include/nuttx/video/rfb.h index 02e15b143f..73f5477611 100644 --- a/include/nuttx/video/rfb.h +++ b/include/nuttx/video/rfb.h @@ -691,6 +691,12 @@ struct rfb_copyrect_encoding_s * the header:" */ +struct rfb_rrehdr8_s +{ + uint8_t nsubrects[4]; /* U32 Number of sub-rectangle */ + uint8_t pixel; /* U8 Background pixel */ +}; + struct rfb_rrehdr16_s { uint8_t nsubrects[4]; /* U32 Number of sub-rectangle */ @@ -707,6 +713,15 @@ struct rfb_rrehdr32_s * structure:" */ +struct rfb_rrerect8_s +{ + uint8_t pixel; /* U8 sub-rect pixel value */ + uint8_t xpos[2]; /* U16 X position */ + uint8_t ypos[2]; /* U16 Y position */ + uint8_t width[2]; /* U16 Width */ + uint8_t height[2]; /* U16 Height */ +}; + struct rfb_rrerect16_s { uint8_t pixel[2]; /* U16 sub-rect pixel value */ @@ -771,6 +786,11 @@ struct rfb_rrerect32_s * background colour for this tile:" */ +struct rfb_backpixel8_s +{ + uint8_t pixel; /* U8 Background pixel value */ +}; + struct rfb_backpixel16_s { uint8_t pixel[2]; /* U16 Background pixel value */ @@ -788,6 +808,11 @@ struct rfb_backpixel32_s * foreground colour to be used for all subrectangles in this tile:" */ +struct rfb_forepixel8_s +{ + uint8_t pixel; /* U8 Foreground pixel value */ +}; + struct rfb_forepixel16_s { uint8_t pixel[2]; /* U16 Foreground pixel value */ @@ -816,6 +841,13 @@ struct rfb_nrects_s * value giving the colour of that subrectangle, so a subrectangle is:" */ +struct rfb_subrectscolored8_s +{ + uint8_t pixel; /* U8 Sub-rect pixel value */ + uint8_t xy; /* U8 X and y position */ + uint8_t wh; /* U8 Width and height */ +}; + struct rfb_subrectscolored16_s { uint8_t pixel[2]; /* U16 Sub-rect pixel value */ @@ -916,6 +948,14 @@ struct rfb_srle_s * height are the width and height of the tile):" */ +struct rfb_rawpixel8_s +{ + uint8_t pixels[1]; /* Actual size is w*h */ +}; + +#define SIZEOF_RFB_RAWPIXEL8_S(n,r) \ + (sizeof(struct rfb_rawpixel8_s) + (n) - 1) + struct rfb_rawpixel16_s { uint8_t pixels[2]; /* Actual size is 2*w*h */ @@ -934,6 +974,11 @@ struct rfb_rawpixel32_s /* "A solid tile consisting of a single colour. The pixel value follows:" */ +struct rfb_solid8_s +{ + uint8_t pixels; /* Pixel value */ +}; + struct rfb_solid16_s { uint8_t pixels[2]; /* Pixel value */ @@ -977,6 +1022,11 @@ struct rfb_solid32_s * (subencoding − 128) pixel values:" */ +struct rfb_palette8_s +{ + uint8_t palette[1]; /* Actual size is palleteSize */ +}; + struct rfb_palette16_s { uint8_t palette[2]; /* Actual size is 2*palleteSize */