From 65f679c7ace526b2d5a6c2b4dba54a2d81838cf0 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Thu, 23 Aug 2012 14:51:57 +0100 Subject: [PATCH] add a line cache add vips_linecache() ... like tilecache, but caches scanlines and sizes dynamically with request size --- ChangeLog | 1 + TODO | 24 -- libvips/conversion/cache.c | 6 - libvips/conversion/conversion.c | 2 + libvips/conversion/old-tilecache.c | 513 +++++++++++++++++++++++++++++ libvips/conversion/sequential.c | 17 +- libvips/conversion/tilecache.c | 328 +++++++++++++----- libvips/deprecated/im_tile_cache.c | 6 - libvips/include/vips/conversion.h | 2 + 9 files changed, 772 insertions(+), 127 deletions(-) create mode 100644 libvips/conversion/old-tilecache.c diff --git a/ChangeLog b/ChangeLog index b80275bb..93017227 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ - fix compiler warnings in production build (thanks Dmitry) - fix spurious warnings about exif updates - VipsSequential has an integrated cache and stalls out of order threads +- add a line cache ... sizes up dynamically with request size 20/7/12 started 7.30.0 - support "rs" mode in vips7 diff --git a/TODO b/TODO index d42d842c..e4e24aca 100644 --- a/TODO +++ b/TODO @@ -1,27 +1,3 @@ -- carrierwave-vips-benchmark seems to fail for 200x200 images with a - non-seq error - - see also the gist comparing oil and vips - - the problem is that sequential access much be a push operation (as chunks of - pixels become available in a certain order), but we are trying to run as a - pull pipeline - - we buffer to minimise mismatches, but that can't always work - - eg. - - $ vips shrink Chicago.png x.jpg 1000 1000 --vips-progress --vips-leak - vips temp-3: 2 threads, 29 x 16 tiles, groups of 768 scanlines - vips temp-3: done in 2.66s - VipsSequential: at line 2176 in file, but line 544 requested - memory: high-water mark 145.19 MB - - - - - - blocking bugs ============= diff --git a/libvips/conversion/cache.c b/libvips/conversion/cache.c index 1ccb9ce4..90190e9a 100644 --- a/libvips/conversion/cache.c +++ b/libvips/conversion/cache.c @@ -49,12 +49,6 @@ #include "conversion.h" -/* Lower and upper bounds for tile cache size. Choose an exact number based on - * tile size. - */ -#define VIPS_MAX_TILE_CACHE (250) -#define VIPS_MIN_TILE_CACHE (5) - typedef struct _VipsCache { VipsConversion parent_instance; diff --git a/libvips/conversion/conversion.c b/libvips/conversion/conversion.c index 16cb659f..b2b8099d 100644 --- a/libvips/conversion/conversion.c +++ b/libvips/conversion/conversion.c @@ -105,6 +105,7 @@ vips_conversion_operation_init( void ) { extern GType vips_copy_get_type( void ); extern GType vips_tile_cache_get_type( void ); + extern GType vips_line_cache_get_type( void ); extern GType vips_sequential_get_type( void ); extern GType vips_cache_get_type( void ); extern GType vips_embed_get_type( void ); @@ -125,6 +126,7 @@ vips_conversion_operation_init( void ) vips_copy_get_type(); vips_tile_cache_get_type(); + vips_line_cache_get_type(); vips_sequential_get_type(); vips_cache_get_type(); vips_embed_get_type(); diff --git a/libvips/conversion/old-tilecache.c b/libvips/conversion/old-tilecache.c new file mode 100644 index 00000000..180f275b --- /dev/null +++ b/libvips/conversion/old-tilecache.c @@ -0,0 +1,513 @@ +/* Tile cache from tiff2vips ... broken out so it can be shared with + * openexr read. + * + * This isn't the same as the sinkscreen cache: we don't sub-divide, and we + * single-thread our callee. + * + * 23/8/06 + * - take ownership of reused tiles in case the cache is being shared + * 13/2/07 + * - release ownership after fillng with pixels in case we read across + * threads + * 4/2/10 + * - gtkdoc + * 12/12/10 + * - use im_prepare_to() and avoid making a sequence for every cache tile + * 5/12/11 + * - rework as a class + * 23/6/12 + * - listen for "minimise" signal + */ + +/* + + This file is part of VIPS. + + VIPS is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a cache of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + */ + +/* + + These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk + + */ + +/* +#define VIPS_DEBUG + */ + +#ifdef HAVE_CONFIG_H +#include +#endif /*HAVE_CONFIG_H*/ +#include + +#include +#include +#include + +#include +#include +#include + +#include "conversion.h" + +/* A tile in our cache. + */ +typedef struct { + struct _VipsTileCache *cache; + + VipsRegion *region; /* Region with private mem for data */ + int time; /* Time of last use for flush */ + int x; /* xy pos in VIPS image cods */ + int y; +} VipsTile; + +typedef struct _VipsTileCache { + VipsConversion parent_instance; + + VipsImage *in; + int tile_width; + int tile_height; + int max_tiles; + VipsCacheStrategy strategy; + + int time; /* Update ticks for LRU here */ + int ntiles; /* Current cache size */ + GMutex *lock; /* Lock everything here */ + GSList *tiles; /* List of tiles */ +} VipsTileCache; + +typedef VipsConversionClass VipsTileCacheClass; + +G_DEFINE_TYPE( VipsTileCache, vips_tile_cache, VIPS_TYPE_CONVERSION ); + +static void +vips_tile_destroy( VipsTile *tile ) +{ + VipsTileCache *cache = tile->cache; + + cache->tiles = g_slist_remove( cache->tiles, tile ); + cache->ntiles -= 1; + g_assert( cache->ntiles >= 0 ); + tile->cache = NULL; + + VIPS_UNREF( tile->region ); + + vips_free( tile ); +} + +static void +vips_tile_cache_drop_all( VipsTileCache *cache ) +{ + while( cache->tiles ) { + VipsTile *tile = (VipsTile *) cache->tiles->data; + + vips_tile_destroy( tile ); + } +} + +static void +vips_tile_cache_dispose( GObject *gobject ) +{ + VipsTileCache *cache = (VipsTileCache *) gobject; + + vips_tile_cache_drop_all( cache ); + VIPS_FREEF( g_mutex_free, cache->lock ); + + G_OBJECT_CLASS( vips_tile_cache_parent_class )->dispose( gobject ); +} + +static VipsTile * +vips_tile_new( VipsTileCache *cache ) +{ + VipsTile *tile; + + if( !(tile = VIPS_NEW( NULL, VipsTile )) ) + return( NULL ); + + tile->cache = cache; + tile->region = NULL; + tile->time = cache->time; + tile->x = -1; + tile->y = -1; + cache->tiles = g_slist_prepend( cache->tiles, tile ); + g_assert( cache->ntiles >= 0 ); + cache->ntiles += 1; + + if( !(tile->region = vips_region_new( cache->in )) ) { + vips_tile_destroy( tile ); + return( NULL ); + } + vips__region_no_ownership( tile->region ); + + return( tile ); +} + +static int +vips_tile_move( VipsTile *tile, int x, int y ) +{ + VipsRect area; + + tile->x = x; + tile->y = y; + + area.left = x; + area.top = y; + area.width = tile->cache->tile_width; + area.height = tile->cache->tile_height; + + if( vips_region_buffer( tile->region, &area ) ) + return( -1 ); + + return( 0 ); +} + +/* Do we have a tile in the cache? + * + * FIXME .. use a hash? + */ +static VipsTile * +vips_tile_search( VipsTileCache *cache, int x, int y ) +{ + GSList *p; + + for( p = cache->tiles; p; p = p->next ) { + VipsTile *tile = (VipsTile *) p->data; + + if( tile->x == x && tile->y == y ) + return( tile ); + } + + return( NULL ); +} + +static void +vips_tile_touch( VipsTile *tile ) +{ + g_assert( tile->cache->ntiles >= 0 ); + + tile->time = tile->cache->time++; +} + +/* Fill a tile with pixels. + */ +static int +vips_tile_fill( VipsTile *tile, VipsRegion *in ) +{ + VipsRect area; + + VIPS_DEBUG_MSG( "tilecache: filling tile %d x %d\n", tile->x, tile->y ); + + area.left = tile->x; + area.top = tile->y; + area.width = tile->cache->tile_width; + area.height = tile->cache->tile_height; + + if( vips_region_prepare_to( in, tile->region, + &area, area.left, area.top ) ) + return( -1 ); + + vips_tile_touch( tile ); + + return( 0 ); +} + +/* Find existing tile, make a new tile, or if we have a full set of tiles, + * reuse LRU. + */ +static VipsTile * +vips_tile_find( VipsTileCache *cache, VipsRegion *in, int x, int y ) +{ + VipsTile *tile; + int oldest, topmost; + GSList *p; + + /* In cache already? + */ + if( (tile = vips_tile_search( cache, x, y )) ) { + vips_tile_touch( tile ); + + return( tile ); + } + + /* VipsTileCache not full? + */ + if( cache->max_tiles == -1 || + cache->ntiles < cache->max_tiles ) { + if( !(tile = vips_tile_new( cache )) || + vips_tile_move( tile, x, y ) || + vips_tile_fill( tile, in ) ) + return( NULL ); + + return( tile ); + } + + /* Reuse an old one. + */ + switch( cache->strategy ) { + case VIPS_CACHE_RANDOM: + oldest = cache->time; + tile = NULL; + for( p = cache->tiles; p; p = p->next ) { + VipsTile *t = (VipsTile *) p->data; + + if( t->time < oldest ) { + oldest = t->time; + tile = t; + } + } + break; + + case VIPS_CACHE_SEQUENTIAL: + topmost = cache->in->Ysize; + tile = NULL; + for( p = cache->tiles; p; p = p->next ) { + VipsTile *t = (VipsTile *) p->data; + + if( t->y < topmost ) { + topmost = t->y; + tile = t; + } + } + break; + + default: + g_assert( 0 ); + } + + g_assert( tile ); + + VIPS_DEBUG_MSG( "tilecache: reusing tile %d x %d\n", tile->x, tile->y ); + + if( vips_tile_move( tile, x, y ) || + vips_tile_fill( tile, in ) ) + return( NULL ); + + return( tile ); +} + +/* Generate func. + */ +static int +vips_tile_cache_gen( VipsRegion *or, + void *seq, void *a, void *b, gboolean *stop ) +{ + VipsRegion *in = (VipsRegion *) seq; + VipsTileCache *cache = (VipsTileCache *) b; + const int tw = cache->tile_width; + const int th = cache->tile_height; + VipsRect *r = &or->valid; + + /* Find top left of tiles we need. + */ + int xs = (r->left / tw) * tw; + int ys = (r->top / th) * th; + + int x, y; + + g_mutex_lock( cache->lock ); + + /* If the output region fits within a tile, we could save a copy by + * routing the output region directly to the tile. + * + * However this would mean that tile drop on minimise could then leave + * dangling pointers, if minimise were called on an active pipeline. + */ + + VIPS_DEBUG_MSG( "vips_tile_cache_gen: " + "left = %d, top = %d, width = %d, height = %d\n", + r->left, r->top, r->width, r->height ); + + for( y = ys; y < VIPS_RECT_BOTTOM( r ); y += th ) + for( x = xs; x < VIPS_RECT_RIGHT( r ); x += tw ) { + VipsTile *tile; + VipsRect tarea; + VipsRect hit; + + if( !(tile = vips_tile_find( cache, in, x, y )) ) { + g_mutex_unlock( cache->lock ); + return( -1 ); + } + + /* The area of the tile. + */ + tarea.left = x; + tarea.top = y; + tarea.width = tw; + tarea.height = th; + + /* The part of the tile that we need. + */ + vips_rect_intersectrect( &tarea, r, &hit ); + vips_region_copy( tile->region, or, &hit, + hit.left, hit.top ); + } + + g_mutex_unlock( cache->lock ); + + return( 0 ); +} + +static void +vips_tile_cache_minimise( VipsImage *image, VipsTileCache *cache ) +{ + vips_tile_cache_drop_all( cache ); +} + +static int +vips_tile_cache_build( VipsObject *object ) +{ + VipsConversion *conversion = VIPS_CONVERSION( object ); + VipsTileCache *cache = (VipsTileCache *) object; + + VIPS_DEBUG_MSG( "vips_tile_cache_build\n" ); + + if( VIPS_OBJECT_CLASS( vips_tile_cache_parent_class )->build( object ) ) + return( -1 ); + + if( vips_image_pio_input( cache->in ) ) + return( -1 ); + + if( vips_image_copy_fields( conversion->out, cache->in ) ) + return( -1 ); + vips_demand_hint( conversion->out, + VIPS_DEMAND_STYLE_SMALLTILE, cache->in, NULL ); + + g_signal_connect( conversion->out, "minimise", + G_CALLBACK( vips_tile_cache_minimise ), cache ); + + if( vips_image_generate( conversion->out, + vips_start_one, vips_tile_cache_gen, vips_stop_one, + cache->in, cache ) ) + return( -1 ); + + return( 0 ); +} + +static void +vips_tile_cache_class_init( VipsTileCacheClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *vobject_class = VIPS_OBJECT_CLASS( class ); + + VIPS_DEBUG_MSG( "vips_tile_cache_class_init\n" ); + + gobject_class->dispose = vips_tile_cache_dispose; + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + vobject_class->nickname = "tilecache"; + vobject_class->description = _( "cache an image" ); + vobject_class->build = vips_tile_cache_build; + + VIPS_ARG_IMAGE( class, "in", 1, + _( "Input" ), + _( "Input image" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsTileCache, in ) ); + + VIPS_ARG_INT( class, "tile_width", 3, + _( "Tile width" ), + _( "Tile width in pixels" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsTileCache, tile_width ), + 1, 1000000, 128 ); + + VIPS_ARG_INT( class, "tile_height", 3, + _( "Tile height" ), + _( "Tile height in pixels" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsTileCache, tile_height ), + 1, 1000000, 128 ); + + VIPS_ARG_INT( class, "max_tiles", 3, + _( "Max tiles" ), + _( "Maximum number of tiles to cache" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsTileCache, max_tiles ), + -1, 1000000, 1000 ); + + VIPS_ARG_ENUM( class, "strategy", 3, + _( "Strategy" ), + _( "Expected access pattern" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsTileCache, strategy ), + VIPS_TYPE_CACHE_STRATEGY, VIPS_CACHE_RANDOM ); +} + +static void +vips_tile_cache_init( VipsTileCache *cache ) +{ + cache->tile_width = 128; + cache->tile_height = 128; + cache->max_tiles = 1000; + cache->strategy = VIPS_CACHE_RANDOM; + + cache->time = 0; + cache->ntiles = 0; + cache->lock = g_mutex_new(); + cache->tiles = NULL; +} + +/** + * vips_tilecache: + * @in: input image + * @out: output image + * @...: %NULL-terminated list of optional named arguments + * + * Optional arguments: + * + * @tile_width: width of tiles in cache + * @tile_height: height of tiles in cache + * @max_tiles: maximum number of tiles to cache + * @strategy: hint expected access pattern #VipsCacheStrategy + * + * This operation behaves rather like vips_copy() between images + * @in and @out, except that it keeps a cache of computed pixels. + * This cache is made of up to @max_tiles tiles (a value of -1 + * means any number of tiles), and each tile is of size @tile_width + * by @tile_height pixels. + * + * Each cache tile is made with a single call to + * vips_image_prepare(). + * + * When the cache fills, a tile is chosen for reuse. If @strategy is + * #VIPS_CACHE_RANDOM, then the least-recently-used tile is reused. If + * @strategy is #VIPS_CACHE_SEQUENTIAL, the top-most tile is reused. + * + * By default, @tile_width and @tile_height are 128 pixels, and the operation + * will cache up to 1,000 tiles. @strategy defaults to #VIPS_CACHE_RANDOM. + * + * This is a lower-level operation than vips_image_cache() since it does no + * subdivision and it single-threads its callee. It is suitable for caching + * the output of operations like exr2vips() on tiled images. + * + * See also: vips_image_cache(). + * + * Returns: 0 on success, -1 on error. + */ +int +vips_tilecache( VipsImage *in, VipsImage **out, ... ) +{ + va_list ap; + int result; + + va_start( ap, out ); + result = vips_call_split( "tilecache", ap, in, out ); + va_end( ap ); + + return( result ); +} diff --git a/libvips/conversion/sequential.c b/libvips/conversion/sequential.c index 38459018..e308625e 100644 --- a/libvips/conversion/sequential.c +++ b/libvips/conversion/sequential.c @@ -12,6 +12,7 @@ * 21/8/12 * - remove skip forward, instead do thread stalling and have an * integrated cache + * - use linecache */ /* @@ -59,10 +60,6 @@ #include "conversion.h" -/* The number of scanlines we cache behind the read point. - */ -#define NLINES (500) - typedef struct _VipsSequential { VipsConversion parent_instance; @@ -108,12 +105,6 @@ vips_sequential_generate( VipsRegion *or, VIPS_DEBUG_MSG( "thread %p request for %d lines from at line %d\n", g_thread_self(), r->height, r->top ); - if( r->height > NLINES ) { - vips_error( "VipsSequential", _( "request for %d scanlines " - "will break caching" ), r->height ); - return( -1 ); - } - if( sequential->trace ) vips_diag( "VipsSequential", "request for %d lines, starting at line %d", @@ -183,13 +174,11 @@ vips_sequential_build( VipsObject *object ) if( vips_image_pio_input( sequential->in ) ) return( -1 ); - if( vips_tilecache( sequential->in, &t, - "tile_width", sequential->in->Xsize, - "tile_height", 1, - "max_tiles", NLINES, + if( vips_linecache( sequential->in, &t, "strategy", VIPS_CACHE_SEQUENTIAL, NULL ) ) return( -1 ); + vips_object_local( object, t ); if( vips_image_copy_fields( conversion->out, t ) ) diff --git a/libvips/conversion/tilecache.c b/libvips/conversion/tilecache.c index 44865229..34ca87e9 100644 --- a/libvips/conversion/tilecache.c +++ b/libvips/conversion/tilecache.c @@ -1,5 +1,4 @@ -/* Tile cache from tiff2vips ... broken out so it can be shared with - * openexr read. +/* Simple tile or line cache. * * This isn't the same as the sinkscreen cache: we don't sub-divide, and we * single-thread our callee. @@ -17,6 +16,8 @@ * - rework as a class * 23/6/12 * - listen for "minimise" signal + * 23/8/12 + * - split to line and tile cache */ /* @@ -64,16 +65,10 @@ #include "conversion.h" -/* Lower and upper bounds for tile cache size. Choose an exact number based on - * tile size. - */ -#define VIPS_MAX_TILE_CACHE (250) -#define VIPS_MIN_TILE_CACHE (5) - /* A tile in our cache. */ typedef struct { - struct _VipsTileCache *cache; + struct _VipsBlockCache *cache; VipsRegion *region; /* Region with private mem for data */ int time; /* Time of last use for flush */ @@ -81,7 +76,7 @@ typedef struct { int y; } VipsTile; -typedef struct _VipsTileCache { +typedef struct _VipsBlockCache { VipsConversion parent_instance; VipsImage *in; @@ -90,22 +85,22 @@ typedef struct _VipsTileCache { int max_tiles; VipsCacheStrategy strategy; - /* VipsTileCache. - */ int time; /* Update ticks for LRU here */ int ntiles; /* Current cache size */ GMutex *lock; /* Lock everything here */ GSList *tiles; /* List of tiles */ -} VipsTileCache; +} VipsBlockCache; -typedef VipsConversionClass VipsTileCacheClass; +typedef VipsConversionClass VipsBlockCacheClass; -G_DEFINE_TYPE( VipsTileCache, vips_tile_cache, VIPS_TYPE_CONVERSION ); +G_DEFINE_TYPE( VipsBlockCache, vips_block_cache, VIPS_TYPE_CONVERSION ); + +#define VIPS_TYPE_BLOCK_CACHE (vips_block_cache_get_type()) static void vips_tile_destroy( VipsTile *tile ) { - VipsTileCache *cache = tile->cache; + VipsBlockCache *cache = tile->cache; cache->tiles = g_slist_remove( cache->tiles, tile ); cache->ntiles -= 1; @@ -118,7 +113,7 @@ vips_tile_destroy( VipsTile *tile ) } static void -vips_tile_cache_drop_all( VipsTileCache *cache ) +vips_block_cache_drop_all( VipsBlockCache *cache ) { while( cache->tiles ) { VipsTile *tile = (VipsTile *) cache->tiles->data; @@ -128,18 +123,18 @@ vips_tile_cache_drop_all( VipsTileCache *cache ) } static void -vips_tile_cache_dispose( GObject *gobject ) +vips_block_cache_dispose( GObject *gobject ) { - VipsTileCache *cache = (VipsTileCache *) gobject; + VipsBlockCache *cache = (VipsBlockCache *) gobject; - vips_tile_cache_drop_all( cache ); + vips_block_cache_drop_all( cache ); VIPS_FREEF( g_mutex_free, cache->lock ); - G_OBJECT_CLASS( vips_tile_cache_parent_class )->dispose( gobject ); + G_OBJECT_CLASS( vips_block_cache_parent_class )->dispose( gobject ); } static VipsTile * -vips_tile_new( VipsTileCache *cache ) +vips_tile_new( VipsBlockCache *cache ) { VipsTile *tile; @@ -188,7 +183,7 @@ vips_tile_move( VipsTile *tile, int x, int y ) * FIXME .. use a hash? */ static VipsTile * -vips_tile_search( VipsTileCache *cache, int x, int y ) +vips_tile_search( VipsBlockCache *cache, int x, int y ) { GSList *p; @@ -237,7 +232,7 @@ vips_tile_fill( VipsTile *tile, VipsRegion *in ) * reuse LRU. */ static VipsTile * -vips_tile_find( VipsTileCache *cache, VipsRegion *in, int x, int y ) +vips_tile_find( VipsBlockCache *cache, VipsRegion *in, int x, int y ) { VipsTile *tile; int oldest, topmost; @@ -251,7 +246,7 @@ vips_tile_find( VipsTileCache *cache, VipsRegion *in, int x, int y ) return( tile ); } - /* VipsTileCache not full? + /* VipsBlockCache not full? */ if( cache->max_tiles == -1 || cache->ntiles < cache->max_tiles ) { @@ -307,16 +302,93 @@ vips_tile_find( VipsTileCache *cache, VipsRegion *in, int x, int y ) return( tile ); } -/* Generate func. +static void +vips_block_cache_minimise( VipsImage *image, VipsBlockCache *cache ) +{ + vips_block_cache_drop_all( cache ); +} + +static int +vips_block_cache_build( VipsObject *object ) +{ + VipsConversion *conversion = VIPS_CONVERSION( object ); + VipsBlockCache *cache = (VipsBlockCache *) object; + + VIPS_DEBUG_MSG( "vips_block_cache_build\n" ); + + if( VIPS_OBJECT_CLASS( vips_block_cache_parent_class )-> + build( object ) ) + return( -1 ); + + g_signal_connect( conversion->out, "minimise", + G_CALLBACK( vips_block_cache_minimise ), cache ); + + return( 0 ); +} + +static void +vips_block_cache_class_init( VipsBlockCacheClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *vobject_class = VIPS_OBJECT_CLASS( class ); + + VIPS_DEBUG_MSG( "vips_block_cache_class_init\n" ); + + gobject_class->dispose = vips_block_cache_dispose; + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + vobject_class->nickname = "blockcache"; + vobject_class->description = _( "cache an image" ); + vobject_class->build = vips_block_cache_build; + + VIPS_ARG_IMAGE( class, "in", 1, + _( "Input" ), + _( "Input image" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsBlockCache, in ) ); + + VIPS_ARG_ENUM( class, "strategy", 3, + _( "Strategy" ), + _( "Expected access pattern" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsBlockCache, strategy ), + VIPS_TYPE_CACHE_STRATEGY, VIPS_CACHE_RANDOM ); +} + +static void +vips_block_cache_init( VipsBlockCache *cache ) +{ + cache->tile_width = 128; + cache->tile_height = 128; + cache->max_tiles = 1000; + cache->strategy = VIPS_CACHE_RANDOM; + + cache->time = 0; + cache->ntiles = 0; + cache->lock = g_mutex_new(); + cache->tiles = NULL; +} + +typedef struct _VipsTileCache { + VipsBlockCache parent_instance; + +} VipsTileCache; + +typedef VipsBlockCacheClass VipsTileCacheClass; + +G_DEFINE_TYPE( VipsTileCache, vips_tile_cache, VIPS_TYPE_BLOCK_CACHE ); + +/* Also called from vips_line_cache_gen(), beware. */ static int vips_tile_cache_gen( VipsRegion *or, void *seq, void *a, void *b, gboolean *stop ) { VipsRegion *in = (VipsRegion *) seq; - VipsTileCache *cache = (VipsTileCache *) b; - const int tw = cache->tile_width; - const int th = cache->tile_height; + VipsBlockCache *block_cache = (VipsBlockCache *) b; + const int tw = block_cache->tile_width; + const int th = block_cache->tile_height; VipsRect *r = &or->valid; /* Find top left of tiles we need. @@ -326,11 +398,10 @@ vips_tile_cache_gen( VipsRegion *or, int x, y; - g_mutex_lock( cache->lock ); + g_mutex_lock( block_cache->lock ); - /* If the output region fits within a tile (tiles can become quite - * large for seq caches, for example), we could save a copy by routing - * the output region directly to the tile. + /* If the output region fits within a tile, we could save a copy by + * routing the output region directly to the tile. * * However this would mean that tile drop on minimise could then leave * dangling pointers, if minimise were called on an active pipeline. @@ -346,8 +417,9 @@ vips_tile_cache_gen( VipsRegion *or, VipsRect tarea; VipsRect hit; - if( !(tile = vips_tile_find( cache, in, x, y )) ) { - g_mutex_unlock( cache->lock ); + if( !(tile = vips_tile_find( block_cache, + in, x, y )) ) { + g_mutex_unlock( block_cache->lock ); return( -1 ); } @@ -365,42 +437,35 @@ vips_tile_cache_gen( VipsRegion *or, hit.left, hit.top ); } - g_mutex_unlock( cache->lock ); + g_mutex_unlock( block_cache->lock ); return( 0 ); } -static void -vips_tile_cache_minimise( VipsImage *image, VipsTileCache *cache ) -{ - vips_tile_cache_drop_all( cache ); -} - static int vips_tile_cache_build( VipsObject *object ) { VipsConversion *conversion = VIPS_CONVERSION( object ); + VipsBlockCache *block_cache = (VipsBlockCache *) object; VipsTileCache *cache = (VipsTileCache *) object; VIPS_DEBUG_MSG( "vips_tile_cache_build\n" ); - if( VIPS_OBJECT_CLASS( vips_tile_cache_parent_class )->build( object ) ) + if( VIPS_OBJECT_CLASS( vips_tile_cache_parent_class )-> + build( object ) ) return( -1 ); - if( vips_image_pio_input( cache->in ) ) + if( vips_image_pio_input( block_cache->in ) ) return( -1 ); - if( vips_image_copy_fields( conversion->out, cache->in ) ) + if( vips_image_copy_fields( conversion->out, block_cache->in ) ) return( -1 ); vips_demand_hint( conversion->out, - VIPS_DEMAND_STYLE_SMALLTILE, cache->in, NULL ); - - g_signal_connect( conversion->out, "minimise", - G_CALLBACK( vips_tile_cache_minimise ), cache ); + VIPS_DEMAND_STYLE_SMALLTILE, block_cache->in, NULL ); if( vips_image_generate( conversion->out, vips_start_one, vips_tile_cache_gen, vips_stop_one, - cache->in, cache ) ) + block_cache->in, cache ) ) return( -1 ); return( 0 ); @@ -414,61 +479,39 @@ vips_tile_cache_class_init( VipsTileCacheClass *class ) VIPS_DEBUG_MSG( "vips_tile_cache_class_init\n" ); - gobject_class->dispose = vips_tile_cache_dispose; gobject_class->set_property = vips_object_set_property; gobject_class->get_property = vips_object_get_property; vobject_class->nickname = "tilecache"; - vobject_class->description = _( "cache an image" ); + vobject_class->description = _( "cache an image as a set of tiles" ); vobject_class->build = vips_tile_cache_build; - VIPS_ARG_IMAGE( class, "in", 1, - _( "Input" ), - _( "Input image" ), - VIPS_ARGUMENT_REQUIRED_INPUT, - G_STRUCT_OFFSET( VipsTileCache, in ) ); - VIPS_ARG_INT( class, "tile_width", 3, _( "Tile width" ), _( "Tile width in pixels" ), VIPS_ARGUMENT_OPTIONAL_INPUT, - G_STRUCT_OFFSET( VipsTileCache, tile_width ), + G_STRUCT_OFFSET( VipsBlockCache, tile_width ), 1, 1000000, 128 ); VIPS_ARG_INT( class, "tile_height", 3, _( "Tile height" ), _( "Tile height in pixels" ), VIPS_ARGUMENT_OPTIONAL_INPUT, - G_STRUCT_OFFSET( VipsTileCache, tile_height ), + G_STRUCT_OFFSET( VipsBlockCache, tile_height ), 1, 1000000, 128 ); VIPS_ARG_INT( class, "max_tiles", 3, _( "Max tiles" ), _( "Maximum number of tiles to cache" ), VIPS_ARGUMENT_OPTIONAL_INPUT, - G_STRUCT_OFFSET( VipsTileCache, max_tiles ), + G_STRUCT_OFFSET( VipsBlockCache, max_tiles ), -1, 1000000, 1000 ); - VIPS_ARG_ENUM( class, "strategy", 3, - _( "Strategy" ), - _( "Expected access pattern" ), - VIPS_ARGUMENT_OPTIONAL_INPUT, - G_STRUCT_OFFSET( VipsTileCache, strategy ), - VIPS_TYPE_CACHE_STRATEGY, VIPS_CACHE_RANDOM ); } static void vips_tile_cache_init( VipsTileCache *cache ) { - cache->tile_width = 128; - cache->tile_height = 128; - cache->max_tiles = 1000; - cache->strategy = VIPS_CACHE_RANDOM; - - cache->time = 0; - cache->ntiles = 0; - cache->lock = g_mutex_new(); - cache->tiles = NULL; } /** @@ -504,7 +547,7 @@ vips_tile_cache_init( VipsTileCache *cache ) * subdivision and it single-threads its callee. It is suitable for caching * the output of operations like exr2vips() on tiled images. * - * See also: vips_image_cache(). + * See also: vips_cache(), vips_linecache(). * * Returns: 0 on success, -1 on error. */ @@ -520,3 +563,134 @@ vips_tilecache( VipsImage *in, VipsImage **out, ... ) return( result ); } + +typedef struct _VipsLineCache { + VipsBlockCache parent_instance; + +} VipsLineCache; + +typedef VipsBlockCacheClass VipsLineCacheClass; + +G_DEFINE_TYPE( VipsLineCache, vips_line_cache, VIPS_TYPE_BLOCK_CACHE ); + +static int +vips_line_cache_gen( VipsRegion *or, + void *seq, void *a, void *b, gboolean *stop ) +{ + VipsBlockCache *block_cache = (VipsBlockCache *) b; + + g_mutex_lock( block_cache->lock ); + + /* We size up the cache to the largest request. + */ + if( or->valid.height > block_cache->max_tiles ) + block_cache->max_tiles = or->valid.height; + + g_mutex_unlock( block_cache->lock ); + + return( vips_tile_cache_gen( or, seq, a, b, stop ) ); +} + +static int +vips_line_cache_build( VipsObject *object ) +{ + VipsConversion *conversion = VIPS_CONVERSION( object ); + VipsBlockCache *block_cache = (VipsBlockCache *) object; + VipsLineCache *cache = (VipsLineCache *) object; + + VIPS_DEBUG_MSG( "vips_line_cache_build\n" ); + + if( VIPS_OBJECT_CLASS( vips_line_cache_parent_class )-> + build( object ) ) + return( -1 ); + + /* Set the cache geometry from the image size ... a set of scanlines. + */ + block_cache->tile_width = block_cache->in->Xsize; + block_cache->tile_height = 1; + + /* This adjust with request size, see vips_line_cache_gen(). + */ + block_cache->max_tiles = 100; + + if( vips_image_pio_input( block_cache->in ) ) + return( -1 ); + + if( vips_image_copy_fields( conversion->out, block_cache->in ) ) + return( -1 ); + vips_demand_hint( conversion->out, + VIPS_DEMAND_STYLE_THINSTRIP, block_cache->in, NULL ); + + if( vips_image_generate( conversion->out, + vips_start_one, vips_line_cache_gen, vips_stop_one, + block_cache->in, cache ) ) + return( -1 ); + + return( 0 ); +} + +static void +vips_line_cache_class_init( VipsLineCacheClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *vobject_class = VIPS_OBJECT_CLASS( class ); + + VIPS_DEBUG_MSG( "vips_line_cache_class_init\n" ); + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + vobject_class->nickname = "linecache"; + vobject_class->description = _( "cache an image as a set of lines" ); + vobject_class->build = vips_line_cache_build; + +} + +static void +vips_line_cache_init( VipsLineCache *cache ) +{ +} + +/** + * vips_linecache: + * @in: input image + * @out: output image + * @...: %NULL-terminated list of optional named arguments + * + * Optional arguments: + * + * @strategy: hint expected access pattern #VipsCacheStrategy + * + * This operation behaves rather like vips_copy() between images + * @in and @out, except that it keeps a cache of computed pixels. + * This cache is made of a set of scanlines. The number of lines cached is + * equal to the maximum prepare request. + * + * Each cache tile is made with a single call to + * vips_image_prepare(). + * + * When the cache fills, a tile is chosen for reuse. If @strategy is + * #VIPS_CACHE_RANDOM, then the least-recently-used tile is reused. If + * @strategy is #VIPS_CACHE_SEQUENTIAL, the top-most tile is reused. + * @strategy defaults to #VIPS_CACHE_RANDOM. + * + * This is a lower-level operation than vips_image_cache() since it does no + * subdivision and it single-threads its callee. It is suitable for caching + * the output of operations like png load. + * + * See also: vips_cache(), vips_tilecache(). + * + * Returns: 0 on success, -1 on error. + */ +int +vips_linecache( VipsImage *in, VipsImage **out, ... ) +{ + va_list ap; + int result; + + va_start( ap, out ); + result = vips_call_split( "linecache", ap, in, out ); + va_end( ap ); + + return( result ); +} diff --git a/libvips/deprecated/im_tile_cache.c b/libvips/deprecated/im_tile_cache.c index b1b54cb0..f5ee9c75 100644 --- a/libvips/deprecated/im_tile_cache.c +++ b/libvips/deprecated/im_tile_cache.c @@ -57,12 +57,6 @@ #include #include -/* Lower and upper bounds for tile cache size. Choose an exact number based on - * tile size. - */ -#define IM_MAX_TILE_CACHE (250) -#define IM_MIN_TILE_CACHE (5) - /* A tile in our cache. */ typedef struct { diff --git a/libvips/include/vips/conversion.h b/libvips/include/vips/conversion.h index 128c8241..3467d200 100644 --- a/libvips/include/vips/conversion.h +++ b/libvips/include/vips/conversion.h @@ -157,6 +157,8 @@ int vips_copy( VipsImage *in, VipsImage **out, ... ) __attribute__((sentinel)); int vips_tilecache( VipsImage *in, VipsImage **out, ... ) __attribute__((sentinel)); +int vips_linecache( VipsImage *in, VipsImage **out, ... ) + __attribute__((sentinel)); int vips_sequential( VipsImage *in, VipsImage **out, ... ) __attribute__((sentinel)); int vips_cache( VipsImage *in, VipsImage **out, ... )