Archipelago » qemu

/*

 * QEMU VNC display driver: tight encoding

 *

 * From libvncserver/libvncserver/tight.c

 * Copyright (C) 2000, 2001 Const Kaplinsky.  All Rights Reserved.

 * Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.

 *

 * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>

 *

 * Permission is hereby granted, free of charge, to any person obtaining a copy

 * of this software and associated documentation files (the "Software"), to deal

 * in the Software without restriction, including without limitation the rights

 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

 * copies of the Software, and to permit persons to whom the Software is

 * furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice shall be included in

 * all copies or substantial portions of the Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

 * THE SOFTWARE.

 */

#include "config-host.h"

/* This needs to be before jpeglib.h line because of conflict with

   INT32 definitions between jmorecfg.h (included by jpeglib.h) and

   Win32 basetsd.h (included by windows.h). */

#include "qemu-common.h"

#ifdef CONFIG_VNC_PNG

/* The following define is needed by pngconf.h. Otherwise it won't compile,

   because setjmp.h was already included by qemu-common.h. */

#define PNG_SKIP_SETJMP_CHECK

#include <png.h>

#endif

#ifdef CONFIG_VNC_JPEG

#include <stdio.h>

#include <jpeglib.h>

#endif

#include "bswap.h"

#include "qint.h"

#include "vnc.h"

#include "vnc-enc-tight.h"

#include "vnc-palette.h"

/* Compression level stuff. The following array contains various

   encoder parameters for each of 10 compression levels (0..9).

   Last three parameters correspond to JPEG quality levels (0..9). */

static const struct {

    int max_rect_size, max_rect_width;

    int mono_min_rect_size, gradient_min_rect_size;

    int idx_zlib_level, mono_zlib_level, raw_zlib_level, gradient_zlib_level;

    int gradient_threshold, gradient_threshold24;

    int idx_max_colors_divisor;

    int jpeg_quality, jpeg_threshold, jpeg_threshold24;

} tight_conf[] = {

    {   512,   32,   6, 65536, 0, 0, 0, 0,   0,   0,   4,  5, 10000, 23000 },

    {  2048,  128,   6, 65536, 1, 1, 1, 0,   0,   0,   8, 10,  8000, 18000 },

    {  6144,  256,   8, 65536, 3, 3, 2, 0,   0,   0,  24, 15,  6500, 15000 },

    { 10240, 1024,  12, 65536, 5, 5, 3, 0,   0,   0,  32, 25,  5000, 12000 },

    { 16384, 2048,  12, 65536, 6, 6, 4, 0,   0,   0,  32, 37,  4000, 10000 },

    { 32768, 2048,  12,  4096, 7, 7, 5, 4, 150, 380,  32, 50,  3000,  8000 },

    { 65536, 2048,  16,  4096, 7, 7, 6, 4, 170, 420,  48, 60,  2000,  5000 },

    { 65536, 2048,  16,  4096, 8, 8, 7, 5, 180, 450,  64, 70,  1000,  2500 },

    { 65536, 2048,  32,  8192, 9, 9, 8, 6, 190, 475,  64, 75,   500,  1200 },

    { 65536, 2048,  32,  8192, 9, 9, 9, 6, 200, 500,  96, 80,   200,   500 }

};

static int tight_send_framebuffer_update(VncState *vs, int x, int y,

                                         int w, int h);

#ifdef CONFIG_VNC_JPEG

static const struct {

    double jpeg_freq_min;       /* Don't send JPEG if the freq is bellow */

    double jpeg_freq_threshold; /* Always send JPEG if the freq is above */

    int jpeg_idx;               /* Allow indexed JPEG */

    int jpeg_full;              /* Allow full color JPEG */

} tight_jpeg_conf[] = {

    { 0,   8,  1, 1 },

    { 0,   8,  1, 1 },

    { 0,   8,  1, 1 },

    { 0,   8,  1, 1 },

    { 0,   10, 1, 1 },

    { 0.1, 10, 1, 1 },

    { 0.2, 10, 1, 1 },

    { 0.3, 12, 0, 0 },

    { 0.4, 14, 0, 0 },

    { 0.5, 16, 0, 0 },

};

#endif

#ifdef CONFIG_VNC_PNG

static const struct {

    int png_zlib_level, png_filters;

} tight_png_conf[] = {

    { 0, PNG_NO_FILTERS },

    { 1, PNG_NO_FILTERS },

    { 2, PNG_NO_FILTERS },

    { 3, PNG_NO_FILTERS },

    { 4, PNG_NO_FILTERS },

    { 5, PNG_ALL_FILTERS },

    { 6, PNG_ALL_FILTERS },

    { 7, PNG_ALL_FILTERS },

    { 8, PNG_ALL_FILTERS },

    { 9, PNG_ALL_FILTERS },

};

static int send_png_rect(VncState *vs, int x, int y, int w, int h,

                         VncPalette *palette);

static bool tight_can_send_png_rect(VncState *vs, int w, int h)

{

    if (vs->tight.type != VNC_ENCODING_TIGHT_PNG) {

        return false;

    }

    if (ds_get_bytes_per_pixel(vs->ds) == 1 ||

        vs->clientds.pf.bytes_per_pixel == 1) {

        return false;

    }

    return true;

}

#endif

/*

 * Code to guess if given rectangle is suitable for smooth image

 * compression (by applying "gradient" filter or JPEG coder).

 */

static unsigned int

tight_detect_smooth_image24(VncState *vs, int w, int h)

{

    int off;

    int x, y, d, dx;

    unsigned int c;

    unsigned int stats[256];

    int pixels = 0;

    int pix, left[3];

    unsigned int errors;

    unsigned char *buf = vs->tight.tight.buffer;

    /*

     * If client is big-endian, color samples begin from the second

     * byte (offset 1) of a 32-bit pixel value.

     */

    off = !!(vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG);

    memset(stats, 0, sizeof (stats));

    for (y = 0, x = 0; y < h && x < w;) {

        for (d = 0; d < h - y && d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH;

             d++) {

            for (c = 0; c < 3; c++) {

                left[c] = buf[((y+d)*w+x+d)*4+off+c] & 0xFF;

            }

            for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; dx++) {

                for (c = 0; c < 3; c++) {

                    pix = buf[((y+d)*w+x+d+dx)*4+off+c] & 0xFF;

                    stats[abs(pix - left[c])]++;

                    left[c] = pix;

                }

                pixels++;

            }

        }

        if (w > h) {

            x += h;

            y = 0;

        } else {

            x = 0;

            y += w;

        }

    }

    /* 95% smooth or more ... */

    if (stats[0] * 33 / pixels >= 95) {

        return 0;

    }

    errors = 0;

    for (c = 1; c < 8; c++) {

        errors += stats[c] * (c * c);

        if (stats[c] == 0 || stats[c] > stats[c-1] * 2) {

            return 0;

        }

    }

    for (; c < 256; c++) {

        errors += stats[c] * (c * c);

    }

    errors /= (pixels * 3 - stats[0]);

    return errors;

}

#define DEFINE_DETECT_FUNCTION(bpp)                                     \

                                                                        \

    static unsigned int                                                 \

    tight_detect_smooth_image##bpp(VncState *vs, int w, int h) {        \

        bool endian;                                                    \

        uint##bpp##_t pix;                                              \

        int max[3], shift[3];                                           \

        int x, y, d, dx;                                                \

        unsigned int c;                                                 \

        unsigned int stats[256];                                        \

        int pixels = 0;                                                 \

        int sample, sum, left[3];                                       \

        unsigned int errors;                                            \

        unsigned char *buf = vs->tight.tight.buffer;                    \

                                                                        \

        endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) !=        \

                  (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG));     \

                                                                        \

                                                                        \

        max[0] = vs->clientds.pf.rmax;                                  \

        max[1] = vs->clientds.pf.gmax;                                  \

        max[2] = vs->clientds.pf.bmax;                                  \

        shift[0] = vs->clientds.pf.rshift;                              \

        shift[1] = vs->clientds.pf.gshift;                              \

        shift[2] = vs->clientds.pf.bshift;                              \

                                                                        \

        memset(stats, 0, sizeof(stats));                                \

                                                                        \

        y = 0, x = 0;                                                   \

        while (y < h && x < w) {                                        \

            for (d = 0; d < h - y &&                                    \

                     d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; d++) {  \

                pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d];              \

                if (endian) {                                           \

                    pix = bswap##bpp(pix);                              \

                }                                                       \

                for (c = 0; c < 3; c++) {                               \

                    left[c] = (int)(pix >> shift[c] & max[c]);          \

                }                                                       \

                for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH;       \

                     dx++) {                                            \

                    pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d+dx];       \

                    if (endian) {                                       \

                        pix = bswap##bpp(pix);                          \

                    }                                                   \

                    sum = 0;                                            \

                    for (c = 0; c < 3; c++) {                           \

                        sample = (int)(pix >> shift[c] & max[c]);       \

                        sum += abs(sample - left[c]);                   \

                        left[c] = sample;                               \

                    }                                                   \

                    if (sum > 255) {                                    \

                        sum = 255;                                      \

                    }                                                   \

                    stats[sum]++;                                       \

                    pixels++;                                           \

                }                                                       \

            }                                                           \

            if (w > h) {                                                \

                x += h;                                                 \

                y = 0;                                                  \

            } else {                                                    \

                x = 0;                                                  \

                y += w;                                                 \

            }                                                           \

        }                                                               \

                                                                        \

        if ((stats[0] + stats[1]) * 100 / pixels >= 90) {               \

            return 0;                                                   \

        }                                                               \

                                                                        \

        errors = 0;                                                     \

        for (c = 1; c < 8; c++) {                                       \

            errors += stats[c] * (c * c);                               \

            if (stats[c] == 0 || stats[c] > stats[c-1] * 2) {           \

                return 0;                                               \

            }                                                           \

        }                                                               \

        for (; c < 256; c++) {                                          \

            errors += stats[c] * (c * c);                               \

        }                                                               \

        errors /= (pixels - stats[0]);                                  \

                                                                        \

        return errors;                                                  \

    }

DEFINE_DETECT_FUNCTION(16)

DEFINE_DETECT_FUNCTION(32)

static int

tight_detect_smooth_image(VncState *vs, int w, int h)

{

    unsigned int errors;

    int compression = vs->tight.compression;

    int quality = vs->tight.quality;

    if (!vs->vd->lossy) {

        return 0;

    }

    if (ds_get_bytes_per_pixel(vs->ds) == 1 ||

        vs->clientds.pf.bytes_per_pixel == 1 ||

        w < VNC_TIGHT_DETECT_MIN_WIDTH || h < VNC_TIGHT_DETECT_MIN_HEIGHT) {

        return 0;

    }

    if (vs->tight.quality != (uint8_t)-1) {

        if (w * h < VNC_TIGHT_JPEG_MIN_RECT_SIZE) {

            return 0;

        }

    } else {

        if (w * h < tight_conf[compression].gradient_min_rect_size) {

            return 0;

        }

    }

    if (vs->clientds.pf.bytes_per_pixel == 4) {

        if (vs->tight.pixel24) {

            errors = tight_detect_smooth_image24(vs, w, h);

            if (vs->tight.quality != (uint8_t)-1) {

                return (errors < tight_conf[quality].jpeg_threshold24);

            }

            return (errors < tight_conf[compression].gradient_threshold24);

        } else {

            errors = tight_detect_smooth_image32(vs, w, h);

        }

    } else {

        errors = tight_detect_smooth_image16(vs, w, h);

    }

    if (quality != -1) {

        return (errors < tight_conf[quality].jpeg_threshold);

    }

    return (errors < tight_conf[compression].gradient_threshold);

}

/*

 * Code to determine how many different colors used in rectangle.

 */

#define DEFINE_FILL_PALETTE_FUNCTION(bpp)                               \

                                                                        \

    static int                                                          \

    tight_fill_palette##bpp(VncState *vs, int x, int y,                 \

                            int max, size_t count,                      \

                            uint32_t *bg, uint32_t *fg,                 \

                            VncPalette **palette) {                     \

        uint##bpp##_t *data;                                            \

        uint##bpp##_t c0, c1, ci;                                       \

        int i, n0, n1;                                                  \

                                                                        \

        data = (uint##bpp##_t *)vs->tight.tight.buffer;                 \

                                                                        \

        c0 = data[0];                                                   \

        i = 1;                                                          \

        while (i < count && data[i] == c0)                              \

            i++;                                                        \

        if (i >= count) {                                               \

            *bg = *fg = c0;                                             \

            return 1;                                                   \

        }                                                               \

                                                                        \

        if (max < 2) {                                                  \

            return 0;                                                   \

        }                                                               \

                                                                        \

        n0 = i;                                                         \

        c1 = data[i];                                                   \

        n1 = 0;                                                         \

        for (i++; i < count; i++) {                                     \

            ci = data[i];                                               \

            if (ci == c0) {                                             \

                n0++;                                                   \

            } else if (ci == c1) {                                      \

                n1++;                                                   \

            } else                                                      \

                break;                                                  \

        }                                                               \

        if (i >= count) {                                               \

            if (n0 > n1) {                                              \

                *bg = (uint32_t)c0;                                     \

                *fg = (uint32_t)c1;                                     \

            } else {                                                    \

                *bg = (uint32_t)c1;                                     \

                *fg = (uint32_t)c0;                                     \

            }                                                           \

            return 2;                                                   \

        }                                                               \

                                                                        \

        if (max == 2) {                                                 \

            return 0;                                                   \

        }                                                               \

                                                                        \

        *palette = palette_new(max, bpp);                               \

        palette_put(*palette, c0);                                      \

        palette_put(*palette, c1);                                      \

        palette_put(*palette, ci);                                      \

                                                                        \

        for (i++; i < count; i++) {                                     \

            if (data[i] == ci) {                                        \

                continue;                                               \

            } else {                                                    \

                ci = data[i];                                           \

                if (!palette_put(*palette, (uint32_t)ci)) {             \

                    return 0;                                           \

                }                                                       \

            }                                                           \

        }                                                               \

                                                                        \

        return palette_size(*palette);                                  \

    }

DEFINE_FILL_PALETTE_FUNCTION(8)

DEFINE_FILL_PALETTE_FUNCTION(16)

DEFINE_FILL_PALETTE_FUNCTION(32)

static int tight_fill_palette(VncState *vs, int x, int y,

                              size_t count, uint32_t *bg, uint32_t *fg,

                              VncPalette **palette)

{

    int max;

    max = count / tight_conf[vs->tight.compression].idx_max_colors_divisor;

    if (max < 2 &&

        count >= tight_conf[vs->tight.compression].mono_min_rect_size) {

        max = 2;

    }

    if (max >= 256) {

        max = 256;

    }

    switch(vs->clientds.pf.bytes_per_pixel) {

    case 4:

        return tight_fill_palette32(vs, x, y, max, count, bg, fg, palette);

    case 2:

        return tight_fill_palette16(vs, x, y, max, count, bg, fg, palette);

    default:

        max = 2;

        return tight_fill_palette8(vs, x, y, max, count, bg, fg, palette);

    }

    return 0;

}

/*

 * Converting truecolor samples into palette indices.

 */

#define DEFINE_IDX_ENCODE_FUNCTION(bpp)                                 \

                                                                        \

    static void                                                         \

    tight_encode_indexed_rect##bpp(uint8_t *buf, int count,             \

                                   VncPalette *palette) {               \

        uint##bpp##_t *src;                                             \

        uint##bpp##_t rgb;                                              \

        int i, rep;                                                     \

        uint8_t idx;                                                    \

                                                                        \

        src = (uint##bpp##_t *) buf;                                    \

                                                                        \

        for (i = 0; i < count; i++) {                                   \

                                                                        \

            rgb = *src++;                                               \

            rep = 0;                                                    \

            while (i < count && *src == rgb) {                          \

                rep++, src++, i++;                                      \

            }                                                           \

            idx = palette_idx(palette, rgb);                            \

            /*                                                          \

             * Should never happen, but don't break everything          \

             * if it does, use the first color instead                  \

             */                                                         \

            if (idx == (uint8_t)-1) {                                   \

                idx = 0;                                                \

            }                                                           \

            while (rep >= 0) {                                          \

                *buf++ = idx;                                           \

                rep--;                                                  \

            }                                                           \

        }                                                               \

    }

DEFINE_IDX_ENCODE_FUNCTION(16)

DEFINE_IDX_ENCODE_FUNCTION(32)

#define DEFINE_MONO_ENCODE_FUNCTION(bpp)                                \

                                                                        \

    static void                                                         \

    tight_encode_mono_rect##bpp(uint8_t *buf, int w, int h,             \

                                uint##bpp##_t bg, uint##bpp##_t fg) {   \

        uint##bpp##_t *ptr;                                             \

        unsigned int value, mask;                                       \

        int aligned_width;                                              \

        int x, y, bg_bits;                                              \

                                                                        \

        ptr = (uint##bpp##_t *) buf;                                    \

        aligned_width = w - w % 8;                                      \

                                                                        \

        for (y = 0; y < h; y++) {                                       \

            for (x = 0; x < aligned_width; x += 8) {                    \

                for (bg_bits = 0; bg_bits < 8; bg_bits++) {             \

                    if (*ptr++ != bg) {                                 \

                        break;                                          \

                    }                                                   \

                }                                                       \

                if (bg_bits == 8) {                                     \

                    *buf++ = 0;                                         \

                    continue;                                           \

                }                                                       \

                mask = 0x80 >> bg_bits;                                 \

                value = mask;                                           \

                for (bg_bits++; bg_bits < 8; bg_bits++) {               \

                    mask >>= 1;                                         \

                    if (*ptr++ != bg) {                                 \

                        value |= mask;                                  \

                    }                                                   \

                }                                                       \

                *buf++ = (uint8_t)value;                                \

            }                                                           \

                                                                        \

            mask = 0x80;                                                \

            value = 0;                                                  \

            if (x >= w) {                                               \

                continue;                                               \

            }                                                           \

                                                                        \

            for (; x < w; x++) {                                        \

                if (*ptr++ != bg) {                                     \

                    value |= mask;                                      \

                }                                                       \

                mask >>= 1;                                             \

            }                                                           \

            *buf++ = (uint8_t)value;                                    \

        }                                                               \

    }

DEFINE_MONO_ENCODE_FUNCTION(8)

DEFINE_MONO_ENCODE_FUNCTION(16)

DEFINE_MONO_ENCODE_FUNCTION(32)

/*

 * ``Gradient'' filter for 24-bit color samples.

 * Should be called only when redMax, greenMax and blueMax are 255.

 * Color components assumed to be byte-aligned.

 */

static void

tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)

{

    uint32_t *buf32;

    uint32_t pix32;

    int shift[3];

    int *prev;

    int here[3], upper[3], left[3], upperleft[3];

    int prediction;

    int x, y, c;

    buf32 = (uint32_t *)buf;

    memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));

    if ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==

        (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)) {

        shift[0] = vs->clientds.pf.rshift;

        shift[1] = vs->clientds.pf.gshift;

        shift[2] = vs->clientds.pf.bshift;

    } else {

        shift[0] = 24 - vs->clientds.pf.rshift;

        shift[1] = 24 - vs->clientds.pf.gshift;

        shift[2] = 24 - vs->clientds.pf.bshift;

    }

    for (y = 0; y < h; y++) {

        for (c = 0; c < 3; c++) {

            upper[c] = 0;

            here[c] = 0;

        }

        prev = (int *)vs->tight.gradient.buffer;

        for (x = 0; x < w; x++) {

            pix32 = *buf32++;

            for (c = 0; c < 3; c++) {

                upperleft[c] = upper[c];

                left[c] = here[c];

                upper[c] = *prev;

                here[c] = (int)(pix32 >> shift[c] & 0xFF);

                *prev++ = here[c];

                prediction = left[c] + upper[c] - upperleft[c];

                if (prediction < 0) {

                    prediction = 0;

                } else if (prediction > 0xFF) {

                    prediction = 0xFF;

                }

                *buf++ = (char)(here[c] - prediction);

            }

        }

    }

}

/*

 * ``Gradient'' filter for other color depths.

 */

#define DEFINE_GRADIENT_FILTER_FUNCTION(bpp)                            \

                                                                        \

    static void                                                         \

    tight_filter_gradient##bpp(VncState *vs, uint##bpp##_t *buf,        \

                               int w, int h) {                          \

        uint##bpp##_t pix, diff;                                        \

        bool endian;                                                    \

        int *prev;                                                      \

        int max[3], shift[3];                                           \

        int here[3], upper[3], left[3], upperleft[3];                   \

        int prediction;                                                 \

        int x, y, c;                                                    \

                                                                        \

        memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));     \

                                                                        \

        endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) !=        \

                  (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG));     \

                                                                        \

        max[0] = vs->clientds.pf.rmax;                                  \

        max[1] = vs->clientds.pf.gmax;                                  \

        max[2] = vs->clientds.pf.bmax;                                  \

        shift[0] = vs->clientds.pf.rshift;                              \

        shift[1] = vs->clientds.pf.gshift;                              \

        shift[2] = vs->clientds.pf.bshift;                              \

                                                                        \

        for (y = 0; y < h; y++) {                                       \

            for (c = 0; c < 3; c++) {                                   \

                upper[c] = 0;                                           \

                here[c] = 0;                                            \

            }                                                           \

            prev = (int *)vs->tight.gradient.buffer;                    \

            for (x = 0; x < w; x++) {                                   \

                pix = *buf;                                             \

                if (endian) {                                           \

                    pix = bswap##bpp(pix);                              \

                }                                                       \

                diff = 0;                                               \

                for (c = 0; c < 3; c++) {                               \

                    upperleft[c] = upper[c];                            \

                    left[c] = here[c];                                  \

                    upper[c] = *prev;                                   \

                    here[c] = (int)(pix >> shift[c] & max[c]);          \

                    *prev++ = here[c];                                  \

                                                                        \

                    prediction = left[c] + upper[c] - upperleft[c];     \

                    if (prediction < 0) {                               \

                        prediction = 0;                                 \

                    } else if (prediction > max[c]) {                   \

                        prediction = max[c];                            \

                    }                                                   \

                    diff |= ((here[c] - prediction) & max[c])           \

                        << shift[c];                                    \

                }                                                       \

                if (endian) {                                           \

                    diff = bswap##bpp(diff);                            \

                }                                                       \

                *buf++ = diff;                                          \

            }                                                           \

        }                                                               \

    }

DEFINE_GRADIENT_FILTER_FUNCTION(16)

DEFINE_GRADIENT_FILTER_FUNCTION(32)

/*

 * Check if a rectangle is all of the same color. If needSameColor is

 * set to non-zero, then also check that its color equals to the

 * *colorPtr value. The result is 1 if the test is successful, and in

 * that case new color will be stored in *colorPtr.

 */

#define DEFINE_CHECK_SOLID_FUNCTION(bpp)                                \

                                                                        \

    static bool                                                         \

    check_solid_tile##bpp(VncState *vs, int x, int y, int w, int h,     \

                          uint32_t* color, bool samecolor)              \

    {                                                                   \

        VncDisplay *vd = vs->vd;                                        \

        uint##bpp##_t *fbptr;                                           \

        uint##bpp##_t c;                                                \

        int dx, dy;                                                     \

                                                                        \

        fbptr = (uint##bpp##_t *)                                       \

            (vd->server->data + y * ds_get_linesize(vs->ds) +           \

             x * ds_get_bytes_per_pixel(vs->ds));                       \

                                                                        \

        c = *fbptr;                                                     \

        if (samecolor && (uint32_t)c != *color) {                       \

            return false;                                               \

        }                                                               \

                                                                        \

        for (dy = 0; dy < h; dy++) {                                    \

            for (dx = 0; dx < w; dx++) {                                \

                if (c != fbptr[dx]) {                                   \

                    return false;                                       \

                }                                                       \

            }                                                           \

            fbptr = (uint##bpp##_t *)                                   \

                ((uint8_t *)fbptr + ds_get_linesize(vs->ds));           \

        }                                                               \

                                                                        \

        *color = (uint32_t)c;                                           \

        return true;                                                    \

    }

DEFINE_CHECK_SOLID_FUNCTION(32)

DEFINE_CHECK_SOLID_FUNCTION(16)

DEFINE_CHECK_SOLID_FUNCTION(8)

static bool check_solid_tile(VncState *vs, int x, int y, int w, int h,

                             uint32_t* color, bool samecolor)

{

    VncDisplay *vd = vs->vd;

    switch(vd->server->pf.bytes_per_pixel) {

    case 4:

        return check_solid_tile32(vs, x, y, w, h, color, samecolor);

    case 2:

        return check_solid_tile16(vs, x, y, w, h, color, samecolor);

    default:

        return check_solid_tile8(vs, x, y, w, h, color, samecolor);

    }

}

static void find_best_solid_area(VncState *vs, int x, int y, int w, int h,

                                 uint32_t color, int *w_ptr, int *h_ptr)

{

    int dx, dy, dw, dh;

    int w_prev;

    int w_best = 0, h_best = 0;

    w_prev = w;

    for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) {

        dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, y + h - dy);

        dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, w_prev);

        if (!check_solid_tile(vs, x, dy, dw, dh, &color, true)) {

            break;

        }

        for (dx = x + dw; dx < x + w_prev;) {

            dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, x + w_prev - dx);

            if (!check_solid_tile(vs, dx, dy, dw, dh, &color, true)) {

                break;

            }

            dx += dw;

        }

        w_prev = dx - x;

        if (w_prev * (dy + dh - y) > w_best * h_best) {

            w_best = w_prev;

            h_best = dy + dh - y;

        }

    }

    *w_ptr = w_best;

    *h_ptr = h_best;

}

static void extend_solid_area(VncState *vs, int x, int y, int w, int h,

                              uint32_t color, int *x_ptr, int *y_ptr,

                              int *w_ptr, int *h_ptr)

{

    int cx, cy;

    /* Try to extend the area upwards. */

    for ( cy = *y_ptr - 1;

          cy >= y && check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true);

          cy-- );

    *h_ptr += *y_ptr - (cy + 1);

    *y_ptr = cy + 1;

    /* ... downwards. */

    for ( cy = *y_ptr + *h_ptr;

          cy < y + h &&

              check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true);

          cy++ );

    *h_ptr += cy - (*y_ptr + *h_ptr);

    /* ... to the left. */

    for ( cx = *x_ptr - 1;

          cx >= x && check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true);

          cx-- );

    *w_ptr += *x_ptr - (cx + 1);

    *x_ptr = cx + 1;

    /* ... to the right. */

    for ( cx = *x_ptr + *w_ptr;

          cx < x + w &&

              check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true);

          cx++ );

    *w_ptr += cx - (*x_ptr + *w_ptr);

}

static int tight_init_stream(VncState *vs, int stream_id,

                             int level, int strategy)

{

    z_streamp zstream = &vs->tight.stream[stream_id];

    if (zstream->opaque == NULL) {

        int err;

        VNC_DEBUG("VNC: TIGHT: initializing zlib stream %d\n", stream_id);

        VNC_DEBUG("VNC: TIGHT: opaque = %p | vs = %p\n", zstream->opaque, vs);

        zstream->zalloc = vnc_zlib_zalloc;

        zstream->zfree = vnc_zlib_zfree;

        err = deflateInit2(zstream, level, Z_DEFLATED, MAX_WBITS,

                           MAX_MEM_LEVEL, strategy);

        if (err != Z_OK) {

            fprintf(stderr, "VNC: error initializing zlib\n");

            return -1;

        }

        vs->tight.levels[stream_id] = level;

        zstream->opaque = vs;

    }

    if (vs->tight.levels[stream_id] != level) {

        if (deflateParams(zstream, level, strategy) != Z_OK) {

            return -1;

        }

        vs->tight.levels[stream_id] = level;

    }

    return 0;

}

static void tight_send_compact_size(VncState *vs, size_t len)

{

    int lpc = 0;

    int bytes = 0;

    char buf[3] = {0, 0, 0};

    buf[bytes++] = len & 0x7F;

    if (len > 0x7F) {

        buf[bytes-1] |= 0x80;

        buf[bytes++] = (len >> 7) & 0x7F;

        if (len > 0x3FFF) {

            buf[bytes-1] |= 0x80;

            buf[bytes++] = (len >> 14) & 0xFF;

        }

    }

    for (lpc = 0; lpc < bytes; lpc++) {

        vnc_write_u8(vs, buf[lpc]);

    }

}

static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,

                               int level, int strategy)

{

    z_streamp zstream = &vs->tight.stream[stream_id];

    int previous_out;

    if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) {

        vnc_write(vs, vs->tight.tight.buffer, vs->tight.tight.offset);

        return bytes;

    }

    if (tight_init_stream(vs, stream_id, level, strategy)) {