From bf65adc92b407e63d1c0f9b549924798b2525459 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Tue, 24 Nov 2015 15:07:23 +0000 Subject: [PATCH 1/7] try skipping blank tiles ... again, but maybe we can be quicker this time --- doc/gtk-doc.make | 20 ++++++----- libvips/foreign/dzsave.c | 73 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/doc/gtk-doc.make b/doc/gtk-doc.make index e7916563..9ccd0b04 100644 --- a/doc/gtk-doc.make +++ b/doc/gtk-doc.make @@ -25,6 +25,7 @@ TARGET_DIR=$(HTML_DIR)/$(DOC_MODULE) SETUP_FILES = \ $(content_files) \ + $(expand_content_files) \ $(DOC_MAIN_SGML_FILE) \ $(DOC_MODULE)-sections.txt \ $(DOC_MODULE)-overrides.txt @@ -86,7 +87,7 @@ GTK_DOC_V_SETUP_0=@echo " DOC Preparing build"; setup-build.stamp: -$(GTK_DOC_V_SETUP)if test "$(abs_srcdir)" != "$(abs_builddir)" ; then \ - files=`echo $(SETUP_FILES) $(expand_content_files) $(DOC_MODULE).types`; \ + files=`echo $(SETUP_FILES) $(DOC_MODULE).types`; \ if test "x$$files" != "x" ; then \ for file in $$files ; do \ destdir=`dirname $(abs_builddir)/$$file`; \ @@ -118,7 +119,7 @@ scan-build.stamp: setup-build.stamp $(HFILE_GLOB) $(CFILE_GLOB) $(GTK_DOC_V_INTROSPECT)if grep -l '^..*$$' $(DOC_MODULE).types > /dev/null 2>&1 ; then \ scanobj_options=""; \ gtkdoc-scangobj 2>&1 --help | grep >/dev/null "\-\-verbose"; \ - if test "$(?)" = "0"; then \ + if test "$$?" = "0"; then \ if test "x$(V)" = "x1"; then \ scanobj_options="--verbose"; \ fi; \ @@ -162,17 +163,17 @@ GTK_DOC_V_XREF=$(GTK_DOC_V_XREF_$(V)) GTK_DOC_V_XREF_=$(GTK_DOC_V_XREF_$(AM_DEFAULT_VERBOSITY)) GTK_DOC_V_XREF_0=@echo " DOC Fixing cross-references"; -html-build.stamp: sgml.stamp $(DOC_MAIN_SGML_FILE) $(content_files) +html-build.stamp: sgml.stamp $(DOC_MAIN_SGML_FILE) $(content_files) $(expand_content_files) $(GTK_DOC_V_HTML)rm -rf html && mkdir html && \ mkhtml_options=""; \ gtkdoc-mkhtml 2>&1 --help | grep >/dev/null "\-\-verbose"; \ - if test "$(?)" = "0"; then \ + if test "$$?" = "0"; then \ if test "x$(V)" = "x1"; then \ mkhtml_options="$$mkhtml_options --verbose"; \ fi; \ fi; \ gtkdoc-mkhtml 2>&1 --help | grep >/dev/null "\-\-path"; \ - if test "$(?)" = "0"; then \ + if test "$$?" = "0"; then \ mkhtml_options="$$mkhtml_options --path=\"$(abs_srcdir)\""; \ fi; \ cd html && gtkdoc-mkhtml $$mkhtml_options $(MKHTML_OPTIONS) $(DOC_MODULE) ../$(DOC_MAIN_SGML_FILE) @@ -194,11 +195,11 @@ GTK_DOC_V_PDF=$(GTK_DOC_V_PDF_$(V)) GTK_DOC_V_PDF_=$(GTK_DOC_V_PDF_$(AM_DEFAULT_VERBOSITY)) GTK_DOC_V_PDF_0=@echo " DOC Building PDF"; -pdf-build.stamp: sgml.stamp $(DOC_MAIN_SGML_FILE) $(content_files) +pdf-build.stamp: sgml.stamp $(DOC_MAIN_SGML_FILE) $(content_files) $(expand_content_files) $(GTK_DOC_V_PDF)rm -f $(DOC_MODULE).pdf && \ mkpdf_options=""; \ gtkdoc-mkpdf 2>&1 --help | grep >/dev/null "\-\-verbose"; \ - if test "$(?)" = "0"; then \ + if test "$$?" = "0"; then \ if test "x$(V)" = "x1"; then \ mkpdf_options="$$mkpdf_options --verbose"; \ fi; \ @@ -223,12 +224,15 @@ clean-local: @if echo $(SCAN_OPTIONS) | grep -q "\-\-rebuild-types" ; then \ rm -f $(DOC_MODULE).types; \ fi + @if echo $(SCAN_OPTIONS) | grep -q "\-\-rebuild-sections" ; then \ + rm -f $(DOC_MODULE)-sections.txt; \ + fi distclean-local: @rm -rf xml html $(REPORT_FILES) $(DOC_MODULE).pdf \ $(DOC_MODULE)-decl-list.txt $(DOC_MODULE)-decl.txt @if test "$(abs_srcdir)" != "$(abs_builddir)" ; then \ - rm -f $(SETUP_FILES) $(expand_content_files) $(DOC_MODULE).types; \ + rm -f $(SETUP_FILES) $(DOC_MODULE).types; \ fi maintainer-clean-local: diff --git a/libvips/foreign/dzsave.c b/libvips/foreign/dzsave.c index cee9bda7..aabde107 100644 --- a/libvips/foreign/dzsave.c +++ b/libvips/foreign/dzsave.c @@ -454,6 +454,12 @@ struct _VipsForeignSaveDz { * Track bytes written here and try to guess when we'll go over. */ size_t bytes_written; + + /* save->background turned into a pixel that matches the image we are + * saving .. used to test for blank tiles. + */ + VipsPel *ink; + }; typedef VipsForeignSaveClass VipsForeignSaveDzClass; @@ -1074,6 +1080,47 @@ tile_name( Layer *layer, int x, int y ) return( out ); } +static gboolean +tile_equal( VipsImage *image, VipsPel * restrict ink ) +{ + const int bytes = VIPS_IMAGE_SIZEOF_PEL( image ); + + VipsRect rect; + VipsRegion *region; + int x, y, b; + + region = vips_region_new( image ); + + /* We know @image is part of a memory buffer, so this will be quick. + */ + rect.left = 0; + rect.top = 0; + rect.width = image->Xsize; + rect.height = image->Ysize; + if( vips_region_prepare( region, &rect ) ) { + g_object_unref( region ); + return( FALSE ); + } + + for( y = 0; y < image->Ysize; y++ ) { + VipsPel * restrict p = VIPS_REGION_ADDR( region, 0, y ); + + for( x = 0; x < image->Xsize; x++ ) { + for( b = 0; b < bytes; b++ ) + if( p[b] != ink[b] ) { + g_object_unref( region ); + return( FALSE ); + } + + p += bytes; + } + } + + g_object_unref( region ); + + return( TRUE ); +} + static int strip_work( VipsThreadState *state, void *a ) { @@ -1094,7 +1141,7 @@ strip_work( VipsThreadState *state, void *a ) printf( "strip_work\n" ); #endif /*DEBUG_VERBOSE*/ - /* If we are centring we may be outside the real pixels. Skip in + /* If we are centering we may be outside the real pixels. Skip in * this case, and the viewer will display blank.png for us. */ if( dz->centre ) { @@ -1127,6 +1174,20 @@ strip_work( VipsThreadState *state, void *a ) state->pos.width, state->pos.height, NULL ) ) return( -1 ); + /* If we are writing a google map pyramid and the tile is equal to the + * background, don't save. The viewer will display blank.png for us. + */ + if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_GOOGLE && + tile_equal( x, dz->ink ) ) { +#ifdef DEBUG_VERBOSE + printf( "strip_work: skipping blank tile %d x %d\n", + state->x / dz->tile_size, + state->y / dz->tile_size ); +#endif /*DEBUG_VERBOSE*/ + + return( 0 ); + } + /* Google tiles need to be padded up to tilesize. */ if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_GOOGLE ) { @@ -1528,6 +1589,16 @@ vips_foreign_save_dz_build( VipsObject *object ) vips_area_unref( VIPS_AREA( background ) ); } + /* We use ink in google mode to check for blank tiles. + */ + if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_GOOGLE ) { + if( !(dz->ink = vips__vector_to_ink( + class->nickname, save->ready, + VIPS_AREA( save->background )->data, NULL, + VIPS_AREA( save->background )->n )) ) + return( -1 ); + } + if( dz->overlap >= dz->tile_size ) { vips_error( "dzsave", "%s", _( "overlap must be less than tile " From e19f9ea0b3fcc5ce54f3cb03f40b8a56393d7b60 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Tue, 24 Nov 2015 17:08:56 +0000 Subject: [PATCH 2/7] working! doesn't seem to slow us down significantly ... before: $ time vips dzsave CMU-1.svs x --layout google --background "243 243 243 255" real 1m1.940s user 2m15.004s sys 0m37.092s after: $ time vips dzsave CMU-1.svs x --layout google --background "243 243 243 255" strip_work: skipping blank tile 176 x 67 strip_work: skipping blank tile 21 x 112 real 1m3.503s user 2m16.012s sys 0m40.328s --- ChangeLog | 1 + libvips/foreign/dzsave.c | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ChangeLog b/ChangeLog index 588672ed..1d13cb78 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,7 @@ - added vips_mapim() ... resample with an index image, plus test - try to improve vips_resize() quality a little more - vips_resize() can do non-square resizes +- dzsave won't write empty tiles in google mode 7/5/15 started 8.1.1 - oop, vips-8.0 wrapper script should be vips-8.1, thanks Danilo diff --git a/libvips/foreign/dzsave.c b/libvips/foreign/dzsave.c index aabde107..56acbff5 100644 --- a/libvips/foreign/dzsave.c +++ b/libvips/foreign/dzsave.c @@ -55,6 +55,8 @@ * - allow zip > 4gb if we have a recent libgsf * 9/9/15 * - better overlap handling, thanks robclouth + * 24/11/15 + * - don't write empty tiles in google mode */ /* @@ -1179,11 +1181,13 @@ strip_work( VipsThreadState *state, void *a ) */ if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_GOOGLE && tile_equal( x, dz->ink ) ) { + g_object_unref( x ); + #ifdef DEBUG_VERBOSE +#endif /*DEBUG_VERBOSE*/ printf( "strip_work: skipping blank tile %d x %d\n", state->x / dz->tile_size, state->y / dz->tile_size ); -#endif /*DEBUG_VERBOSE*/ return( 0 ); } @@ -1589,16 +1593,6 @@ vips_foreign_save_dz_build( VipsObject *object ) vips_area_unref( VIPS_AREA( background ) ); } - /* We use ink in google mode to check for blank tiles. - */ - if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_GOOGLE ) { - if( !(dz->ink = vips__vector_to_ink( - class->nickname, save->ready, - VIPS_AREA( save->background )->data, NULL, - VIPS_AREA( save->background )->n )) ) - return( -1 ); - } - if( dz->overlap >= dz->tile_size ) { vips_error( "dzsave", "%s", _( "overlap must be less than tile " @@ -1633,6 +1627,16 @@ vips_foreign_save_dz_build( VipsObject *object ) save->ready = z; } + /* We use ink in google mode to check for blank tiles. + */ + if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_GOOGLE ) { + if( !(dz->ink = vips__vector_to_ink( + class->nickname, save->ready, + VIPS_AREA( save->background )->data, NULL, + VIPS_AREA( save->background )->n )) ) + return( -1 ); + } + /* How much we step by as we write tiles. */ dz->tile_step = dz->tile_size - dz->overlap; From 8a31df3214c78557419f9dc789bb2d294c6414e5 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Tue, 26 Apr 2016 10:06:59 +0100 Subject: [PATCH 3/7] only save tiles more than 5 from blank looks for max absolute difference --- libvips/foreign/dzsave.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libvips/foreign/dzsave.c b/libvips/foreign/dzsave.c index c57bb852..05e8f404 100644 --- a/libvips/foreign/dzsave.c +++ b/libvips/foreign/dzsave.c @@ -56,7 +56,7 @@ * 9/9/15 * - better overlap handling, thanks robclouth * 24/11/15 - * - don't write empty tiles in google mode + * - don't write almost blank tiles in google mode * 25/11/15 * - always strip tile metadata * 16/12/15 @@ -1104,6 +1104,12 @@ tile_name( Layer *layer, int x, int y ) return( out ); } +/* Test for tile equal to background colour. In google maps mode, we skip + * blank background tiles. + * + * Don't use exactly equality, since compression artefacts or noise can upset + * this. + */ static gboolean tile_equal( VipsImage *image, VipsPel * restrict ink ) { @@ -1131,7 +1137,7 @@ tile_equal( VipsImage *image, VipsPel * restrict ink ) for( x = 0; x < image->Xsize; x++ ) { for( b = 0; b < bytes; b++ ) - if( p[b] != ink[b] ) { + if( VIPS_ABS( p[b] - ink[b] ) > 5 ) { g_object_unref( region ); return( FALSE ); } @@ -1206,10 +1212,10 @@ strip_work( VipsThreadState *state, void *a ) g_object_unref( x ); #ifdef DEBUG_VERBOSE -#endif /*DEBUG_VERBOSE*/ printf( "strip_work: skipping blank tile %d x %d\n", state->x / dz->tile_size, state->y / dz->tile_size ); +#endif /*DEBUG_VERBOSE*/ return( 0 ); } From 24a51098b178b67eb8051d04cb1cdcfa365d8af8 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Thu, 12 May 2016 14:54:28 +0100 Subject: [PATCH 4/7] compiler warning --- TODO | 2 -- libvips/iofuncs/header.c | 15 ++++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/TODO b/TODO index d8dfc955..cd1ab676 100644 --- a/TODO +++ b/TODO @@ -3,8 +3,6 @@ $ vips copy k2.jpg x.dz[suffix=.jpg[Q=90]] dzsave: not , or ) after parameter -- add n_pages to pdfload - - add more webp tests to py suite - the gif tests in the suite sometimes fail with giflib5 because of an diff --git a/libvips/iofuncs/header.c b/libvips/iofuncs/header.c index 5e913759..61d5b177 100644 --- a/libvips/iofuncs/header.c +++ b/libvips/iofuncs/header.c @@ -433,10 +433,10 @@ vips_image_guess_format( const VipsImage *image ) break; case VIPS_INTERPRETATION_CMYK: - if( image->BandFmt != VIPS_FORMAT_USHORT ) - format = VIPS_FORMAT_UCHAR; + if( image->BandFmt == VIPS_FORMAT_USHORT ) + format = VIPS_FORMAT_USHORT; else - format = image->BandFmt; + format = VIPS_FORMAT_UCHAR; break; case VIPS_INTERPRETATION_LABQ: @@ -453,14 +453,15 @@ vips_image_guess_format( const VipsImage *image ) break; case VIPS_INTERPRETATION_MATRIX: - if( image->BandFmt != VIPS_FORMAT_DOUBLE ) - format = VIPS_FORMAT_FLOAT; + if( image->BandFmt == VIPS_FORMAT_DOUBLE ) + format = VIPS_FORMAT_DOUBLE; else - format = image->BandFmt; + format = VIPS_FORMAT_FLOAT; break; default: - g_assert_not_reached(); + format = VIPS_FORMAT_NOTSET; + break; } return( format ); From 2cfe4842d3b16470f10504e475a2a41abf42e66d Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Sun, 15 May 2016 09:48:02 +0100 Subject: [PATCH 5/7] notes --- TODO | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TODO b/TODO index cd1ab676..6692c7f0 100644 --- a/TODO +++ b/TODO @@ -5,6 +5,8 @@ - add more webp tests to py suite +- try moving some more of the CLI tests to py + - the gif tests in the suite sometimes fail with giflib5 because of an uninitialized struct in giflib, see From 46f2776d22372dde26ad37526cb068261f2261f3 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Sun, 15 May 2016 09:50:40 +0100 Subject: [PATCH 6/7] fix up changelog order --- ChangeLog | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index e051c1fb..b7208d26 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ 1/5/16 started 8.4 - many more wepsave options [Felix Bünemann] - added quant_table option to wepsave [Felix Bünemann] +- dzsave won't write empty tiles in google mode 15/4/16 started 8.3.1 - rename vips wrapper script, it was still vips-8.2, thanks Benjamin @@ -85,9 +86,6 @@ - added vips_mapim() ... resample with an index image, plus test - try to improve vips_resize() quality a little more - vips_resize() can do non-square resizes -<<<<<<< HEAD -- dzsave won't write empty tiles in google mode -======= - dzsave removes tile metadata by default, thanks Benjamin - jpeg strip option removes a little more, thanks Benjamin - added vips_image_new_from_memory_copy() @@ -100,7 +98,6 @@ - add --properties flag to tiffsave - dzsave defaults changed: now writes 256x256 jpegs for non-edge tiles, thanks Daniel ->>>>>>> master 7/5/15 started 8.1.1 - oop, vips-8.0 wrapper script should be vips-8.1, thanks Danilo From 7f02e843dcefb377f6c3f7c8b04060502bb7c6a3 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Sun, 15 May 2016 09:53:37 +0100 Subject: [PATCH 7/7] add ack for dzsave tile skip see https://github.com/jcupitt/libvips/issues/352 --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b7208d26..c107d704 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,8 @@ 1/5/16 started 8.4 - many more wepsave options [Felix Bünemann] - added quant_table option to wepsave [Felix Bünemann] -- dzsave won't write empty tiles in google mode +- dzsave won't write empty tiles in google mode, thanks bverem, perog, + felixbuenemann 15/4/16 started 8.3.1 - rename vips wrapper script, it was still vips-8.2, thanks Benjamin