From 7f94fb359750e9ea26a8f4017ac68aa4c3e7d751 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Wed, 15 Feb 2012 15:47:43 +0000 Subject: [PATCH] break vips_sequential() out --- ChangeLog | 1 + TODO | 2 + libvips/conversion/Makefile.am | 1 + libvips/conversion/conversion.c | 3 +- libvips/conversion/sequential.c | 186 ++++++++++++++++++++++++++ libvips/foreign/foreign.c | 208 +++++++----------------------- libvips/foreign/vipspng.c | 44 +++---- libvips/include/vips/conversion.h | 2 + 8 files changed, 259 insertions(+), 188 deletions(-) create mode 100644 libvips/conversion/sequential.c diff --git a/ChangeLog b/ChangeLog index 94bbe374..59864f0b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ - support operations with many returns in Python - sequential read mode - better im_shrink() +- added vips_sequential() 20/8/11 started 7.27.0 - version bump for new dev cycle diff --git a/TODO b/TODO index 00f60fb8..8a42ada5 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,8 @@ tiff, jpg are the obvious ones + test progressive png, non-sequential png load + - argh (nip2:11576): GLib-GObject-WARNING **: value "-0.000000" of type diff --git a/libvips/conversion/Makefile.am b/libvips/conversion/Makefile.am index c81994f2..ffed0a3c 100644 --- a/libvips/conversion/Makefile.am +++ b/libvips/conversion/Makefile.am @@ -4,6 +4,7 @@ libconversion_la_SOURCES = \ conversion.c \ conversion.h \ tilecache.c \ + sequential.c \ cache.c \ copy.c \ embed.c \ diff --git a/libvips/conversion/conversion.c b/libvips/conversion/conversion.c index 985be1b6..4db7b772 100644 --- a/libvips/conversion/conversion.c +++ b/libvips/conversion/conversion.c @@ -105,6 +105,7 @@ vips_conversion_operation_init( void ) { extern GType vips_copy_get_type( void ); extern GType vips_tile_cache_get_type( void ); + extern GType vips_sequential_get_type( void ); extern GType vips_cache_get_type( void ); extern GType vips_embed_get_type( void ); extern GType vips_flip_get_type( void ); @@ -123,6 +124,7 @@ vips_conversion_operation_init( void ) vips_copy_get_type(); vips_tile_cache_get_type(); + vips_sequential_get_type(); vips_cache_get_type(); vips_embed_get_type(); vips_flip_get_type(); @@ -140,7 +142,6 @@ vips_conversion_operation_init( void ) vips_bandmean_get_type(); } - /* The common part of most binary conversion * operators. We: * diff --git a/libvips/conversion/sequential.c b/libvips/conversion/sequential.c new file mode 100644 index 00000000..7b17e201 --- /dev/null +++ b/libvips/conversion/sequential.c @@ -0,0 +1,186 @@ +/* Like copy, but ensure sequential access. + * + * Handy with sequential for loading files formats which are strictly + * top-to-bottom, like PNG. + * + * 15/2/12 + * - from VipsForeignLoad + */ + +/* + + This file is part of VIPS. + + VIPS is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a cache of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + */ + +/* + + These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk + + */ + +/* +#define VIPS_DEBUG + */ + +#ifdef HAVE_CONFIG_H +#include +#endif /*HAVE_CONFIG_H*/ +#include + +#include +#include +#include + +#include +#include +#include + +#include "conversion.h" + +typedef struct _VipsSequential { + VipsConversion parent_instance; + + VipsImage *in; + + int y_pos; +} VipsSequential; + +typedef VipsConversionClass VipsSequentialClass; + +G_DEFINE_TYPE( VipsSequential, vips_sequential, VIPS_TYPE_CONVERSION ); + +static int +vips_sequential_generate( VipsRegion *or, + void *seq, void *a, void *b, gboolean *stop ) +{ + VipsSequential *sequential = (VipsSequential *) b; + VipsRect *r = &or->valid; + VipsRegion *ir = (VipsRegion *) seq; + + /* The y pos of the request must be the same as our current file + * position. + */ + if( r->top != sequential->y_pos ) { + vips_error( "VipsSequential", + _( "non-sequential read --- " + "at position %d in file, but position %d requested" ), + sequential->y_pos, r->top ); + return( -1 ); + } + + /* We're inside a tilecache where tiles are the full image width, so + * this should always be true. + */ + g_assert( r->left == 0 ); + g_assert( r->width == or->im->Xsize ); + g_assert( VIPS_RECT_BOTTOM( r ) <= or->im->Ysize ); + + /* Pointer copy. + */ + if( vips_region_prepare( ir, r ) || + vips_region_region( or, ir, r, r->left, r->top ) ) + return( -1 ); + + sequential->y_pos += r->height; + + return( 0 ); +} + +static int +vips_sequential_build( VipsObject *object ) +{ + VipsConversion *conversion = VIPS_CONVERSION( object ); + VipsSequential *sequential = (VipsSequential *) object; + + VIPS_DEBUG_MSG( "vips_sequential_build\n" ); + + if( VIPS_OBJECT_CLASS( vips_sequential_parent_class )->build( object ) ) + return( -1 ); + + if( vips_image_pio_input( sequential->in ) ) + return( -1 ); + + if( vips_image_copy_fields( conversion->out, sequential->in ) ) + return( -1 ); + vips_demand_hint( conversion->out, + VIPS_DEMAND_STYLE_FATSTRIP, sequential->in, NULL ); + + if( vips_image_generate( conversion->out, + vips_start_one, vips_sequential_generate, vips_stop_one, + sequential->in, sequential ) ) + return( -1 ); + + return( 0 ); +} + +static void +vips_sequential_class_init( VipsSequentialClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *vobject_class = VIPS_OBJECT_CLASS( class ); + + VIPS_DEBUG_MSG( "vips_sequential_class_init\n" ); + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + vobject_class->nickname = "sequential"; + vobject_class->description = _( "check sequential access" ); + vobject_class->build = vips_sequential_build; + + VIPS_ARG_IMAGE( class, "in", 1, + _( "Input" ), + _( "Input image" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsSequential, in ) ); +} + +static void +vips_sequential_init( VipsSequential *sequential ) +{ +} + +/** + * vips_sequential: + * @in: input image + * @out: output image + * @...: %NULL-terminated list of optional named arguments + * + * This operation behaves rather like vips_copy() between images + * @in and @out, except that it checks that pixels are only requested + * top-to-bottom. If an out of order request is made, it throws an exception. + * + * This operation is handy with tilecache for loading file formats which are + * strictly top-to-bottom, like PNG. + * + * See also: vips_image_cache(). + * + * Returns: 0 on success, -1 on error. + */ +int +vips_sequential( VipsImage *in, VipsImage **out, ... ) +{ + va_list ap; + int result; + + va_start( ap, out ); + result = vips_call_split( "sequential", ap, in, out ); + va_end( ap ); + + return( result ); +} diff --git a/libvips/foreign/foreign.c b/libvips/foreign/foreign.c index 5f4e8c8d..e4dc7a4b 100644 --- a/libvips/foreign/foreign.c +++ b/libvips/foreign/foreign.c @@ -678,6 +678,43 @@ vips_get_disc_threshold( void ) return( threshold ); } +static VipsImage * +vips_foreign_load_temp( VipsForeignLoad *load ) +{ + const guint64 disc_threshold = vips_get_disc_threshold(); + const guint64 image_size = VIPS_IMAGE_SIZEOF_IMAGE( load->out ); + + VipsImage *temp; + + /* We open via disc if: + * - 'disc' is set + * - disc-threshold has not been set to zero + * - the uncompressed image will be larger than + * vips_get_disc_threshold() + */ + if( load->disc && + disc_threshold && + image_size > disc_threshold ) { +#ifdef DEBUG + printf( "vips_foreign_load_temp: making disc temp\n" ); +#endif /*DEBUG*/ + + if( !(temp = vips_image_new_disc_temp( "%s.v" )) ) + return( NULL ); + } + else { +#ifdef DEBUG + printf( "vips_foreign_load_start: making 'p' temp\n" ); +#endif /*DEBUG*/ + + /* Otherwise, fall back to a "p". + */ + temp = vips_image_new(); + } + + return( temp ); +} + /* Check two images for compatibility: their geometries need to match. */ static gboolean @@ -700,43 +737,14 @@ vips_foreign_load_iscompat( VipsImage *a, VipsImage *b ) * on the new image. */ static void * -vips_foreign_load_start( VipsImage *out, void *a, void *dummy ) +vips_foreign_load_start( VipsImage *out, void *a, void *b ) { - VipsForeignLoad *load = VIPS_FOREIGN_LOAD( a ); - VipsForeignLoadClass *class = VIPS_FOREIGN_LOAD_GET_CLASS( a ); + VipsForeignLoad *load = VIPS_FOREIGN_LOAD( b ); + VipsForeignLoadClass *class = VIPS_FOREIGN_LOAD_GET_CLASS( load ); if( !load->real ) { - const guint64 disc_threshold = vips_get_disc_threshold(); - const guint64 image_size = VIPS_IMAGE_SIZEOF_IMAGE( load->out ); - - /* We open via disc if: - * - 'disc' is set - * - disc-threshold has not been set to zero - * - the format does not support lazy read - * - the uncompressed image will be larger than - * vips_get_disc_threshold() - */ - if( load->disc && - disc_threshold && - !(load->flags & VIPS_FOREIGN_PARTIAL) && - image_size > disc_threshold ) { -#ifdef DEBUG - printf( "vips_foreign_load_start: making disc temp\n" ); -#endif /*DEBUG*/ - - if( !(load->real = vips_image_new_disc_temp( "%s.v" )) ) - return( NULL ); - } - - /* Otherwise, fall back to a "p". - */ - if( !load->real ) { -#ifdef DEBUG - printf( "vips_foreign_load_start: making 'p' temp\n" ); -#endif /*DEBUG*/ - - load->real = vips_image_new(); - } + if( !(load->real = vips_foreign_load_temp( load )) ) + return( NULL ); #ifdef DEBUG printf( "vips_foreign_load_start: triggering ->load()\n" ); @@ -785,73 +793,6 @@ vips_foreign_load_generate( VipsRegion *or, return( 0 ); } -static int -vips_foreign_load_seq_generate( VipsRegion *or, - void *seq, void *a, void *b, gboolean *stop ) -{ - VipsForeignLoad *load = (VipsForeignLoad *) b; - VipsRegion *ir = (VipsRegion *) seq; - VipsRect *r = &or->valid; - - /* The y pos of the request must be the same as our current file - * position. - */ - if( r->top != load->y_pos ) { - vips_error( "VipsForeignLoad", - _( "non-sequential read --- " - "at position %d in file, but position %d requested" ), - load->y_pos, r->top ); - return( -1 ); - } - - /* We're inside a tilecache where tiles are the fill image width, so - * this should always be true. - */ - g_assert( r->left == 0 ); - g_assert( r->width == or->im->Xsize ); - g_assert( VIPS_RECT_BOTTOM( r ) <= or->im->Ysize ); - - /* Ask for input we need. - */ - if( vips_region_prepare( ir, r ) ) - return( -1 ); - - /* Attach output region to that. - */ - if( vips_region_region( or, ir, r, r->left, r->top ) ) - return( -1 ); - - load->y_pos += r->height; - - return( 0 ); -} - -/* Like vips_copy(), but check sequentiality of access. - */ -static int -vips_foreign_load_seq( VipsForeignLoad *load, VipsImage *in, VipsImage **out ) -{ - VipsImage *seq; - - seq = vips_image_new(); - vips_demand_hint( seq, VIPS_DEMAND_STYLE_FATSTRIP, - in, NULL ); - if( vips_image_pio_input( in ) || - vips_image_copy_fields( seq, in ) || - vips_image_generate( seq, - vips_start_one, - vips_foreign_load_seq_generate, - vips_stop_one, - in, load ) ) { - g_object_unref( seq ); - return( -1 ); - } - - *out = seq; - - return( 0 ); -} - static int vips_foreign_load_build( VipsObject *object ) { @@ -898,19 +839,16 @@ vips_foreign_load_build( VipsObject *object ) * everything. Otherwise, it's just set fields and we must also * load pixels. * - * Three modes: + * If it's a partial loader, or if it's a sequiential loader and + * sequential load has been requested, we can load here. * - * PARTIAL: just load and copy to @out. This is rather unlikely, most - * partial readers would just define a ->header() method I suppose. - * - * SEQUENTIAL: we need a tile cache plus a thing - * to check sequentiality. - * - * ELSE: it's a write-line thing. We delay the load until the - * first read, and allocate a memory/disk buffer as required. + * Otherwise, it's a whole-image read and we delay until first pixel + * access. */ if( class->load && - (load->flags & VIPS_FOREIGN_PARTIAL) ) { + ((load->flags & VIPS_FOREIGN_PARTIAL) || + ((load->flags & VIPS_FOREIGN_SEQUENTIAL) && + load->sequential)) ) { #ifdef DEBUG printf( "vips_foreign_load_build: partial read\n" ); #endif /*DEBUG*/ @@ -940,54 +878,6 @@ vips_foreign_load_build( VipsObject *object ) load->real, load ) ) return( -1 ); } - else if( class->load && - (load->flags & VIPS_FOREIGN_SEQUENTIAL) ) { - VipsImage **t = (VipsImage **) - vips_object_local_array( VIPS_OBJECT( load ), 2 ); - -#ifdef DEBUG - printf( "vips_foreign_load_build: sequential read\n" ); -#endif /*DEBUG*/ - - /* Load to @real. - */ - load->real = vips_image_new(); - if( class->load( load ) || - vips_image_pio_input( load->real ) ) - return( -1 ); - - /* Must match ->header(). - */ - if( !vips_foreign_load_iscompat( load->real, load->out ) ) - return( -1 ); - - /* Copy to seq, checking sequentiality of accesses. - */ - if( vips_foreign_load_seq( load, load->real, &t[0] ) ) - return( -1 ); - - /* Copy again, with a cache. Enough tiles for two complete - * rows. - */ - if( vips_tilecache( t[0], &t[1], - "tile_width", load->real->Xsize, - "tile_height", VIPS__TILE_HEIGHT, - "max_tiles", 2, - NULL ) ) - return( -1 ); - - /* Finally, copy to out. - */ - vips_demand_hint( load->out, load->out->dhint, t[1], NULL ); - - if( vips_image_pio_input( load->real ) || - vips_image_generate( load->out, - vips_start_one, - vips_foreign_load_generate, - vips_stop_one, - t[1], load ) ) - return( -1 ); - } else if( class->load ) { #ifdef DEBUG printf( "vips_foreign_load_build: whole-image read\n" ); diff --git a/libvips/foreign/vipspng.c b/libvips/foreign/vipspng.c index 70f3e4d6..b460938c 100644 --- a/libvips/foreign/vipspng.c +++ b/libvips/foreign/vipspng.c @@ -65,8 +65,8 @@ */ /* -#define DEBUG */ +#define DEBUG #ifdef HAVE_CONFIG_H #include @@ -352,13 +352,6 @@ png2vips_generate( VipsRegion *or, if( setjmp( png_jmpbuf( read->pPng ) ) ) return( -1 ); - /* We're inside a tilecache where tiles are the fill image width, so - * this should always be true. - */ - g_assert( r->left == 0 ); - g_assert( r->width == or->im->Xsize ); - g_assert( VIPS_RECT_BOTTOM( r ) <= or->im->Ysize ); - for( y = 0; y < r->height; y++ ) { png_bytep q = (png_bytep) VIPS_REGION_ADDR( or, 0, r->top + y ); @@ -368,24 +361,6 @@ png2vips_generate( VipsRegion *or, return( 0 ); } -/* Build with vips_image_generate(), but fail if out-of-order tiles are - * requested. - * - * We are behind a tile cache, so we can assume that vips_image_generate() - * will always be asked for tiles which are the full image width. We can also - * assume we are single-threaded. And that we will be called sequentially. - */ -static int -png2vips_sequential( Read *read, VipsImage *out ) -{ - if( vips_image_generate( out, - NULL, png2vips_generate, NULL, - read, NULL ) ) - return( -1 ); - - return( 0 ); -} - /* Interlaced PNGs need to be entirely decompressed into memory then can be * served partially from there. Non-interlaced PNGs may be read sequentially. */ @@ -425,8 +400,21 @@ vips__png_read( const char *name, VipsImage *out ) return( -1 ); } else { - if( png2vips_header( read, out ) || - png2vips_sequential( read, out ) ) + VipsImage **t = (VipsImage **) + vips_object_local_array( VIPS_OBJECT( out ), 3 ); + + t[0] = vips_image_new(); + if( png2vips_header( read, t[0] ) || + vips_image_generate( t[0], + NULL, png2vips_generate, NULL, + read, NULL ) || + vips_sequential( t[0], &t[1], NULL ) || + vips_tilecache( t[1], &t[2], + "tile_width", t[0]->Xsize, + "tile_height", VIPS__TILE_HEIGHT, + "max_tiles", 2, + NULL ) || + vips_image_write( t[2], out ) ) return( -1 ); } diff --git a/libvips/include/vips/conversion.h b/libvips/include/vips/conversion.h index 3a26de0f..bb46f6fb 100644 --- a/libvips/include/vips/conversion.h +++ b/libvips/include/vips/conversion.h @@ -139,6 +139,8 @@ int vips_copy( VipsImage *in, VipsImage **out, ... ) __attribute__((sentinel)); int vips_tilecache( VipsImage *in, VipsImage **out, ... ) __attribute__((sentinel)); +int vips_sequential( VipsImage *in, VipsImage **out, ... ) + __attribute__((sentinel)); int vips_cache( VipsImage *in, VipsImage **out, ... ) __attribute__((sentinel));