sergiotarxz
/
libvips
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remove stalling

This commit is contained in:
John Cupitt 2017-02-21 15:03:15 +00:00
parent c6eb9ee0f2
commit 959f412380
42 changed files with 103 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
TODO
View File

@ -1,58 +1,17 @@
- we currently have
- seq no longer stalls ahead threads ... instead, we rely on sinkdisk
interlocks to limit how far ahead or behind threads can get
* seq is on line 1000
* thread1 asks for line 2000 and blocks on seq->ready
* thread2 asks for line 2001 and also blocks
* thread1 times out and skips ahead to 2000
* thread1 wakes up thread2
* thread2 reads 2001
deprecate UNBUFFERED? it will probably not work now
we could have
remove single-thread-first-tile thing
* seq is on line 1000
* thread1 asks for line 2000 and blocks on seq->ready
* thread2 asks for line 2001
* thread2 sees that another thread is blocked earlier, so there has
probably been a skip
* thread2 sets a "skipahead" flag, then wakes up thread1 and sleeps
itself
* thread1 wakes, skips to 2000, wakes thread2 and returns
* thread2 wakes, reads 2001, and returns
the readbehind option to many loaders is no longer needed
we could keep a list of all the blocked threads, sorted by y request, then
only unblock the lowest numbered one when the whole threadpool is waiting
do we need to restore the UNBUFFERED enum member to keep binary compat? we
need to make sure old code using UNBUFF now uses regular SEQ (and not RAND)
this sounds like the safest idea
we could get rid of the first tile is single-threaded hack as well
argh no, we may not ever get more than one thread blocked on a seq
we just have to get rid of seq and rely on a larger buffer behind the
processing point
use sinkdisk / sinkmemory to limit the amount that threads can get out of
sync ... we must stop starting new threads if an old thread is taking too
long to produce its tile
sinkdisk will do this automatically, since it only keeps two lines of tiles
sinkmemory will need to have something added
so ... just swap vips_sequentiial for vips_linecache, look at adding
interlocks to sinkmemory, test, possibly increase margins
what will do skipahead now? we have to build this into linecache ... or maybe
leave seq there and just strip out the stalling and sync code
make tiff2vips track and verify y_pos
foreign operations affected
tiff2vips.c
jpeg2vips.c
radiance.c
vipspng.c
- vips linecache has access there twice!

2
libvips/arithmetic/arithmetic.c
View File

@ -633,7 +633,7 @@ vips_arithmetic_class_init( VipsArithmeticClass *class )
vobject_class->description = _( "arithmetic operations" );
vobject_class->build = vips_arithmetic_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "out", 100,
_( "Output" ),

2
libvips/arithmetic/statistic.c
View File

@ -161,7 +161,7 @@ vips_statistic_class_init( VipsStatisticClass *class )
vobject_class->description = _( "VIPS statistic operations" );
vobject_class->build = vips_statistic_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 0,
_( "Input" ),

2
libvips/colour/colour.c
View File

@ -421,7 +421,7 @@ vips_colour_class_init( VipsColourClass *class )
vobject_class->description = _( "color operations" );
vobject_class->build = vips_colour_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "out", 100,
_( "Output" ),

2
libvips/colour/colourspace.c
View File

@ -580,7 +580,7 @@ vips_colourspace_class_init( VipsColourspaceClass *class )
vobject_class->description = _( "convert to a new colorspace" );
vobject_class->build = vips_colourspace_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/colour/sRGB2scRGB.c
View File

@ -237,7 +237,7 @@ vips_sRGB2scRGB_class_init( VipssRGB2scRGBClass *class )
object_class->description = _( "convert an sRGB image to scRGB" );
object_class->build = vips_sRGB2scRGB_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/colour/scRGB2BW.c
View File

@ -231,7 +231,7 @@ vips_scRGB2BW_class_init( VipsscRGB2BWClass *class )
object_class->description = _( "convert scRGB to BW" );
object_class->build = vips_scRGB2BW_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/colour/scRGB2sRGB.c
View File

@ -259,7 +259,7 @@ vips_scRGB2sRGB_class_init( VipsscRGB2sRGBClass *class )
object_class->description = _( "convert an scRGB image to sRGB" );
object_class->build = vips_scRGB2sRGB_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/arrayjoin.c
View File

@ -298,7 +298,7 @@ vips_arrayjoin_class_init( VipsArrayjoinClass *class )
vobject_class->description = _( "join an array of images" );
vobject_class->build = vips_arrayjoin_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_BOXED( class, "in", -1,
_( "Input" ),

2
libvips/conversion/bandary.c
View File

@ -186,7 +186,7 @@ vips_bandary_class_init( VipsBandaryClass *class )
vobject_class->description = _( "operations on image bands" );
vobject_class->build = vips_bandary_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
}
static void

2
libvips/conversion/bandfold.c
View File

@ -154,7 +154,7 @@ vips_bandfold_class_init( VipsBandfoldClass *class )
vobject_class->description = _( "fold up x axis into bands" );
vobject_class->build = vips_bandfold_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/bandunfold.c
View File

@ -157,7 +157,7 @@ vips_bandunfold_class_init( VipsBandunfoldClass *class )
vobject_class->description = _( "unfold image bands into x axis" );
vobject_class->build = vips_bandunfold_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/byteswap.c
View File

@ -200,7 +200,7 @@ vips_byteswap_class_init( VipsByteswapClass *class )
vobject_class->description = _( "byteswap an image" );
vobject_class->build = vips_byteswap_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/cast.c
View File

@ -542,7 +542,7 @@ vips_cast_class_init( VipsCastClass *class )
vobject_class->description = _( "cast an image" );
vobject_class->build = vips_cast_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

3
libvips/conversion/copy.c
View File

@ -264,8 +264,7 @@ vips_copy_class_init( VipsCopyClass *class )
* cache it. Plus copy is cheap.
*/
operation_class->flags =
VIPS_OPERATION_SEQUENTIAL_UNBUFFERED |
VIPS_OPERATION_NOCACHE;
VIPS_OPERATION_SEQUENTIAL | VIPS_OPERATION_NOCACHE;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/embed.c
View File

@ -557,7 +557,7 @@ vips_embed_class_init( VipsEmbedClass *class )
vobject_class->description = _( "embed an image in a larger image" );
vobject_class->build = vips_embed_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", -1,
_( "Input" ),

2
libvips/conversion/extract.c
View File

@ -193,7 +193,7 @@ vips_extract_area_class_init( VipsExtractAreaClass *class )
vobject_class->description = _( "extract an area from an image" );
vobject_class->build = vips_extract_area_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "input", 0,
_( "Input" ),

2
libvips/conversion/falsecolour.c
View File

@ -375,7 +375,7 @@ vips_falsecolour_class_init( VipsFalsecolourClass *class )
vobject_class->description = _( "false-color an image" );
vobject_class->build = vips_falsecolour_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 0,
_( "in" ),

2
libvips/conversion/flatten.c
View File

@ -380,7 +380,7 @@ vips_flatten_class_init( VipsFlattenClass *class )
vobject_class->description = _( "flatten alpha out of an image" );
vobject_class->build = vips_flatten_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/gamma.c
View File

@ -137,7 +137,7 @@ vips_gamma_class_init( VipsGammaClass *class )
vobject_class->description = _( "gamma an image" );
vobject_class->build = vips_gamma_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", -1,
_( "in" ),

2
libvips/conversion/insert.c
View File

@ -509,8 +509,6 @@ vips_insert_class_init( VipsInsertClass *class )
_( "insert image @sub into @main at @x, @y" );
vobject_class->build = vips_insert_build;
/* Can't be UNBUFFERED, we are a SMALLTILE operation.
*/
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "main", -1,

2
libvips/conversion/msb.c
View File

@ -238,7 +238,7 @@ vips_msb_class_init( VipsMsbClass *class )
_( "pick most-significant byte from an image" );
vobject_class->build = vips_msb_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 0,
_( "Input" ),

2
libvips/conversion/premultiply.c
View File

@ -253,7 +253,7 @@ vips_premultiply_class_init( VipsPremultiplyClass *class )
vobject_class->description = _( "premultiply image alpha" );
vobject_class->build = vips_premultiply_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/recomb.c
View File

@ -201,7 +201,7 @@ vips_recomb_class_init( VipsRecombClass *class )
object_class->description = _( "linear recombination with matrix" );
object_class->build = vips_recomb_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 0,
_( "Input" ),

135
libvips/conversion/sequential.c
View File

@ -21,6 +21,8 @@
* this broke on some busy, many-core systems, see comment below
* 10/6/14
* - re-enable skipahead now we have the single-thread-first-tile idea
* 21/2/17
* - remove stalling logic, deprecate access option
*/
/*
@ -51,7 +53,6 @@
*/
/*
#define VIPS_DEBUG_GREEN
#define VIPS_DEBUG
*/
@ -70,28 +71,17 @@
#include "pconversion.h"
/* Stall threads that run ahead for up to this long, in seconds. Normally they
* will be woken once their data is ready and long before this. The timeout is
* just to prevent a total crash in the case of accidental deadlock.
*
* This has to be a long time: if we're trying to use all cores on a busy
* system, it could be ages until all the other threads get a chance to run.
*/
#define STALL_TIME (60.0)
typedef struct _VipsSequential {
VipsConversion parent_instance;
VipsImage *in;
int tile_height;
VipsAccess access;
gboolean trace;
/* Lock access to y_pos with this, use the cond to wake up stalled
* threads.
*/
GMutex *lock;
GCond *ready;
/* The next read from our source will fetch this scanline, ie. it's 0
* when we start.
@ -102,6 +92,10 @@ typedef struct _VipsSequential {
* can stall and never wake.
*/
int error;
/* Deprecated.
*/
VipsAccess access;
} VipsSequential;
typedef VipsConversionClass VipsSequentialClass;
@ -114,7 +108,6 @@ vips_sequential_dispose( GObject *gobject )
VipsSequential *sequential = (VipsSequential *) gobject;
VIPS_FREEF( vips_g_mutex_free, sequential->lock );
VIPS_FREEF( vips_g_cond_free, sequential->ready );
G_OBJECT_CLASS( vips_sequential_parent_class )->dispose( gobject );
}
@ -140,8 +133,6 @@ vips_sequential_generate( VipsRegion *or,
VIPS_GATE_STOP( "vips_sequential_generate: wait" );
VIPS_DEBUG_MSG_GREEN( "thread %p has lock ...\n", g_thread_self() );
/* If we've seen an error, everything must stop.
*/
if( sequential->error ) {
@ -149,87 +140,18 @@ vips_sequential_generate( VipsRegion *or,
return( -1 );
}
if( r->top > sequential->y_pos &&
sequential->y_pos > 0 ) {
/* This request is for stuff beyond the current read position,
* and this is not the first request. We
* stall for a while to give other threads time to catch up.
*
* The stall can be cancelled by a signal on @ready.
*
* We don't stall forever, since an error would be better than
* deadlock, and we don't fail on timeout, since the timeout
* may be harmless.
*/
#ifdef HAVE_COND_INIT
gint64 time;
time = g_get_monotonic_time() +
STALL_TIME * G_TIME_SPAN_SECOND;
#else
GTimeVal time;
g_get_current_time( &time );
g_time_val_add( &time, STALL_TIME * 1000000 );
#endif
VIPS_DEBUG_MSG_GREEN( "thread %p stalling for up to %gs ...\n",
g_thread_self(), STALL_TIME );
VIPS_GATE_START( "vips_sequential_generate: wait" );
/* Exit the loop on timeout or condition passes. We have to
* be wary of spurious wakeups.
*/
while( r->top > sequential->y_pos ) {
#ifdef HAVE_COND_INIT
if( !g_cond_wait_until( sequential->ready,
sequential->lock, time ) )
break;
#else
if( !g_cond_timed_wait( sequential->ready,
sequential->lock, &time ) )
break;
#endif
/* We may have woken up because of an eval error.
*/
if( sequential->error ) {
g_mutex_unlock( sequential->lock );
return( -1 );
}
}
VIPS_GATE_STOP( "vips_sequential_generate: wait" );
VIPS_DEBUG_MSG_GREEN( "thread %p awake again ...\n",
g_thread_self() );
}
if( r->top > sequential->y_pos ) {
/* This is a request for something some way down the image,
* and we've fallen through from the stall above.
*
* Probably the operation is something like extract_area and
* we should skip the initial part of the image. In fact,
* we read to cache, since it may be useful.
/* This is a request for something some way down the image.
* Skip down the image, saving any pixels in cache.
*/
VipsRect area;
VIPS_DEBUG_MSG_GREEN( "thread %p skipping to line %d ...\n",
g_thread_self(),
r->top );
area.left = 0;
area.top = sequential->y_pos;
area.width = 1;
area.height = r->top - sequential->y_pos;
if( vips_region_prepare( ir, &area ) ) {
VIPS_DEBUG_MSG( "thread %p error, unlocking #1 ...\n",
g_thread_self() );
sequential->error = -1;
g_cond_broadcast( sequential->ready );
g_mutex_unlock( sequential->lock );
return( -1 );
}
@ -237,36 +159,18 @@ vips_sequential_generate( VipsRegion *or,
sequential->y_pos = VIPS_RECT_BOTTOM( &area );
}
/* This is a request for old or present pixels -- serve from cache.
* This may trigger further, sequential reads.
/* This is a request for old or present pixels.
*/
VIPS_DEBUG_MSG_GREEN( "thread %p reading ...\n", g_thread_self() );
if( vips_region_prepare( ir, r ) ||
vips_region_region( or, ir, r, r->left, r->top ) ) {
VIPS_DEBUG_MSG( "thread %p error, unlocking #2 ...\n",
g_thread_self() );
sequential->error = -1;
g_cond_broadcast( sequential->ready );
g_mutex_unlock( sequential->lock );
return( -1 );
}
if( VIPS_RECT_BOTTOM( r ) > sequential->y_pos ) {
/* This request has moved the read point. Update it, and wake
* up all stalled threads for a retry.
*/
if( VIPS_RECT_BOTTOM( r ) > sequential->y_pos )
sequential->y_pos = VIPS_RECT_BOTTOM( r );
VIPS_DEBUG_MSG_GREEN( "thread %p updating y_pos to %d and "
"waking stalled\n",
g_thread_self(),
sequential->y_pos );
g_cond_broadcast( sequential->ready );
}
VIPS_DEBUG_MSG_GREEN( "thread %p unlocking ...\n", g_thread_self() );
g_mutex_unlock( sequential->lock );
return( 0 );
@ -290,7 +194,7 @@ vips_sequential_build( VipsObject *object )
if( vips_linecache( sequential->in, &t,
"tile_height", sequential->tile_height,
"access", sequential->access,
"access", VIPS_ACCESS_SEQUENTIAL,
NULL ) )
return( -1 );
@ -346,7 +250,7 @@ vips_sequential_class_init( VipsSequentialClass *class )
VIPS_ARG_ENUM( class, "access", 6,
_( "Strategy" ),
_( "Expected access pattern" ),
VIPS_ARGUMENT_OPTIONAL_INPUT,
VIPS_ARGUMENT_OPTIONAL_INPUT | VIPS_ARGUMENT_DEPRECATED,
G_STRUCT_OFFSET( VipsSequential, access ),
VIPS_TYPE_ACCESS, VIPS_ACCESS_SEQUENTIAL );
}
@ -355,9 +259,7 @@ static void
vips_sequential_init( VipsSequential *sequential )
{
sequential->trace = FALSE;
sequential->access = VIPS_ACCESS_SEQUENTIAL;
sequential->lock = vips_g_mutex_new();
sequential->ready = vips_g_cond_new();
sequential->tile_height = 1;
sequential->error = 0;
}
@ -372,12 +274,11 @@ vips_sequential_init( VipsSequential *sequential )
*
* * @trace: trace requests
* * @strip_height: height of cache strips
* * @access: access pattern
*
* This operation behaves rather like vips_copy() between images
* @in and @out, except that it checks that pixels are only requested
* top-to-bottom. If a thread makes an out of order request, it is stalled
* until the pack catches up.
* @in and @out, except that it make sure that pixels from in are read strictly
* top-to-bottom. A line cache is used to save some scanlines behind the read
* point, so that some degree of out of order requests is OK.
*
* This operation is useful for loading file formats which are
* strictly top-to-bottom, like PNG.
@ -387,10 +288,8 @@ vips_sequential_init( VipsSequential *sequential )
* non-sequential accesses.
*
* @strip_height can be used to set the size of the tiles that
* vips_sequential() uses. The default value is 1.
*
* @access can be set to #VIPS_ACCESS_SEQUENTIAL_UNBUFFERED, meaning don't
* keep a large cache behind the read point. This can save some memory.
* vips_sequential() uses. The default value is 1. It can be set (for example)
* to the strip size for TIFF for a speedup.
*
* See also: vips_cache(), vips_linecache(), vips_tilecache().
*

47
libvips/conversion/tilecache.c
View File

@ -280,7 +280,6 @@ vips_tile_search_recycle( gpointer key, gpointer value, gpointer user_data )
break;
case VIPS_ACCESS_SEQUENTIAL:
case VIPS_ACCESS_SEQUENTIAL_UNBUFFERED:
if( tile->pos.top < search->topmost ) {
search->topmost = tile->pos.top;
search->tile = tile;
@ -842,7 +841,7 @@ vips_tile_cache_init( VipsTileCache *cache )
*
* When the cache fills, a tile is chosen for reuse. If @access is
* #VIPS_ACCESS_RANDOM, then the least-recently-used tile is reused. If
* @access is #VIPS_ACCESS_SEQUENTIAL or #VIPS_ACCESS_SEQUENTIAL_UNBUFFERED,
* @access is #VIPS_ACCESS_SEQUENTIAL,
* the top-most tile is reused.
*
* By default, @tile_width and @tile_height are 128 pixels, and the operation
@ -917,6 +916,10 @@ vips_line_cache_build( VipsObject *object )
VipsBlockCache *block_cache = (VipsBlockCache *) object;
VipsLineCache *cache = (VipsLineCache *) object;
int tile_width;
int tile_height;
int n_lines;
VIPS_DEBUG_MSG( "vips_line_cache_build\n" );
if( VIPS_OBJECT_CLASS( vips_line_cache_parent_class )->
@ -929,31 +932,18 @@ vips_line_cache_build( VipsObject *object )
block_cache->access = cache->access;
if( cache->access == VIPS_ACCESS_SEQUENTIAL_UNBUFFERED )
/* A tile per thread.
*
* Imagine scanline tiles and four threads. And add a bit for
* slop.
*/
block_cache->max_tiles = 2 * vips_concurrency_get();
else {
/* Enough lines for two complete buffers would be exactly
* right. Make it 4 to give us some slop room.
*
* This can go up with request size, see vips_line_cache_gen().
*/
int tile_width;
int tile_height;
int n_lines;
/* Enough lines for two complete buffers would be exactly
* right. Make it 4 to give us some slop room.
*
* This can go up with request size, see vips_line_cache_gen().
*/
vips_get_tile_size( block_cache->in,
&tile_width, &tile_height, &n_lines );
block_cache->max_tiles = 4 *
(1 + n_lines / block_cache->tile_height);
vips_get_tile_size( block_cache->in,
&tile_width, &tile_height, &n_lines );
block_cache->max_tiles = 4 *
(1 + n_lines / block_cache->tile_height);
VIPS_DEBUG_MSG( "vips_line_cache_build: n_lines = %d\n",
n_lines );
}
VIPS_DEBUG_MSG( "vips_line_cache_build: n_lines = %d\n",
n_lines );
VIPS_DEBUG_MSG( "vips_line_cache_build: "
"max_tiles = %d, tile_height = %d\n",
@ -1026,15 +1016,14 @@ vips_line_cache_init( VipsLineCache *cache )
* @in and @out, except that it keeps a cache of computed scanlines.
*
* The number of lines cached is enough for a small amount of non-local
* access. If you know you will not be making any non-local access, you can
* save some memory and set @access to #VIPS_ACCESS_SEQUENTIAL_UNBUFFERED.
* access.
*
* Each cache tile is made with a single call to
* vips_region_prepare().
*
* When the cache fills, a tile is chosen for reuse. If @access is
* #VIPS_ACCESS_RANDOM, then the least-recently-used tile is reused. If
* @access is #VIPS_ACCESS_SEQUENTIAL or #VIPS_ACCESS_SEQUENTIAL_UNBUFFERED,
* @access is #VIPS_ACCESS_SEQUENTIAL,
* the top-most tile is reused. @access defaults to #VIPS_ACCESS_RANDOM.
*
* @tile_height can be used to set the size of the strips that

2
libvips/conversion/unpremultiply.c
View File

@ -262,7 +262,7 @@ vips_unpremultiply_class_init( VipsUnpremultiplyClass *class )
vobject_class->description = _( "unpremultiply image alpha" );
vobject_class->build = vips_unpremultiply_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

2
libvips/conversion/zoom.c
View File

@ -377,7 +377,7 @@ vips_zoom_class_init( VipsZoomClass *class )
vobject_class->description = _( "zoom an image" );
vobject_class->build = vips_zoom_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "input", 0,
_( "Input" ),

2
libvips/foreign/foreign.c
View File

@ -1502,7 +1502,7 @@ vips_foreign_save_class_init( VipsForeignSaveClass *class )
* write and interlaced png write, which are not, add extra caches
* on their input.
*/
operation_class->flags |= VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags |= VIPS_OPERATION_SEQUENTIAL;
/* Must not cache savers.
*/

7
libvips/foreign/jpeg2vips.c
View File

@ -552,7 +552,7 @@ read_jpeg_generate( VipsRegion *or,
g_assert( r->height == VIPS_MIN( 8, or->im->Ysize - r->top ) );
/* And check that y_pos is correct. It should be, since we are inside
* a vips_linecache().
* a vips_sequential().
*/
if( r->top != jpeg->y_pos ) {
VIPS_GATE_STOP( "read_jpeg_generate: work" );
@ -674,11 +674,8 @@ read_jpeg_image( ReadJpeg *jpeg, VipsImage *out )
if( vips_image_generate( t[0],
NULL, read_jpeg_generate, NULL,
jpeg, NULL ) ||
vips_linecache( t[0], &t[1],
vips_sequential( t[0], &t[1],
"tile_height", 8,
"access", jpeg->readbehind ?
VIPS_ACCESS_SEQUENTIAL :
VIPS_ACCESS_SEQUENTIAL_UNBUFFERED,
NULL ) )
return( -1 );

5
libvips/foreign/radiance.c
View File

@ -1155,11 +1155,8 @@ vips__rad_load( const char *filename, VipsImage *out, gboolean readbehind )
if( vips_image_generate( t[0],
NULL, rad2vips_generate, NULL,
read, NULL ) ||
vips_linecache( t[0], &t[1],
vips_sequential( t[0], &t[1],
"tile_height", 8,
"access", readbehind ?
VIPS_ACCESS_SEQUENTIAL :
VIPS_ACCESS_SEQUENTIAL_UNBUFFERED,
NULL ) ||
vips_image_write( t[1], out ) )
return( -1 );

22
libvips/foreign/tiff2vips.c
View File

@ -306,6 +306,11 @@ typedef struct _Rtiff {
* strips or tiles interleaved.
*/
tdata_t contig_buf;
/* Track the current y read position in strip mode to make sure we're
* being called correctly.
*/
int y_pos;
} Rtiff;
/* Test for field exists.
@ -1731,6 +1736,16 @@ rtiff_stripwise_generate( VipsRegion *or,
g_assert( r->height ==
VIPS_MIN( rows_per_strip, or->im->Ysize - r->top ) );
/* And check that y_pos is correct. It should be, since we are inside
* a vips_sequential().
*/
if( r->top != rtiff->y_pos ) {
vips_error( "tiff2vips",
_( "out of order read at line %d" ), rtiff->y_pos );
return( -1 );
}
VIPS_GATE_START( "rtiff_stripwise_generate: work" );
y = 0;
@ -1826,6 +1841,7 @@ rtiff_stripwise_generate( VipsRegion *or,
}
y += hit.height;
rtiff->y_pos += hit.height;
}
VIPS_GATE_STOP( "rtiff_stripwise_generate: work" );
@ -1924,11 +1940,8 @@ rtiff_read_stripwise( Rtiff *rtiff, VipsImage *out )
vips_image_generate( t[0],
NULL, rtiff_stripwise_generate, NULL,
rtiff, NULL ) ||
vips_linecache( t[0], &t[1],
vips_sequential( t[0], &t[1],
"tile_height", rtiff->header.rows_per_strip,
"access", rtiff->readbehind ?
VIPS_ACCESS_SEQUENTIAL :
VIPS_ACCESS_SEQUENTIAL_UNBUFFERED,
NULL ) ||
rtiff_autorotate( rtiff, t[1], &t[2] ) ||
vips_image_write( t[2], out ) )
@ -1973,6 +1986,7 @@ rtiff_new( VipsImage *out,
rtiff->memcpy = FALSE;
rtiff->plane_buf = NULL;
rtiff->contig_buf = NULL;
rtiff->y_pos = 0;
g_signal_connect( out, "close",
G_CALLBACK( rtiff_close ), rtiff );

7
libvips/foreign/vipspng.c
View File

@ -497,7 +497,7 @@ png2vips_generate( VipsRegion *or,
g_assert( r->height == VIPS_MIN( 8, or->im->Ysize - r->top ) );
/* And check that y_pos is correct. It should be, since we are inside
* a vips_linecache().
* a vips_sequential().
*/
if( r->top != read->y_pos ) {
vips_error( "vipspng",
@ -590,11 +590,8 @@ png2vips_image( Read *read, VipsImage *out )
vips_image_generate( t[0],
NULL, png2vips_generate, NULL,
read, NULL ) ||
vips_linecache( t[0], &t[1],
vips_sequential( t[0], &t[1],
"tile_height", 8,
"access", read->readbehind ?
VIPS_ACCESS_SEQUENTIAL :
VIPS_ACCESS_SEQUENTIAL_UNBUFFERED,
NULL ) ||
vips_image_write( t[1], out ) )
return( -1 );

2
libvips/histogram/maplut.c
View File

@ -688,7 +688,7 @@ vips_maplut_class_init( VipsMaplutClass *class )
object_class->description = _( "map an image though a lut" );
object_class->build = vips_maplut_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_IMAGE( class, "in", 1,
_( "Input" ),

5
libvips/include/vips/image.h
View File

@ -129,10 +129,13 @@ typedef enum {
typedef enum {
VIPS_ACCESS_RANDOM,
VIPS_ACCESS_SEQUENTIAL,
VIPS_ACCESS_SEQUENTIAL_UNBUFFERED,
VIPS_ACCESS_LAST
} VipsAccess;
/* Compat for a mode we used to support.
*/
#define VIPS_ACCESS_SEQUENTIAL_UNBUFFERED VIPS_ACCESS_SEQUENTIAL
struct _VipsImage;
struct _VipsRegion;

1
libvips/iofuncs/enumtypes.c
View File

@ -669,7 +669,6 @@ vips_access_get_type( void )
static const GEnumValue values[] = {
{VIPS_ACCESS_RANDOM, "VIPS_ACCESS_RANDOM", "random"},
{VIPS_ACCESS_SEQUENTIAL, "VIPS_ACCESS_SEQUENTIAL", "sequential"},
{VIPS_ACCESS_SEQUENTIAL_UNBUFFERED, "VIPS_ACCESS_SEQUENTIAL_UNBUFFERED", "sequential-unbuffered"},
{VIPS_ACCESS_LAST, "VIPS_ACCESS_LAST", "last"},
{0, NULL, NULL}
};

8
libvips/iofuncs/image.c
View File

@ -147,7 +147,6 @@
* VipsAccess:
* @VIPS_ACCESS_RANDOM: can read anywhere
* @VIPS_ACCESS_SEQUENTIAL: top-to-bottom reading only, but with a small buffer
* @VIPS_ACCESS_SEQUENTIAL_UNBUFFERED: top-to-bottom reading only
*
* The type of access an operation has to supply. See vips_tilecache()
* and #VipsForeign.
@ -156,9 +155,6 @@
*
* @VIPS_ACCESS_SEQUENTIAL means requests will be top-to-bottom, but with some
* amount of buffering behind the read point for small non-local accesses.
*
* @VIPS_ACCESS_SEQUENTIAL_UNBUFFERED means requests will be strictly
* top-to-bottom with no read-behind. This can save some memory.
*/
/**
@ -1856,9 +1852,7 @@ vips_filename_get_options( const char *vips_filename )
* whole image exactly once, top-to-bottom. In this mode, vips can avoid
* converting the whole image in one go, for a large memory saving. You are
* allowed to make small non-local references, so area operations like
* convolution will work. #VIPS_ACCESS_SEQUENTIAL_UNBUFFERED does not allow
* non-local references, so will only work for very strict top-to-bottom
* operations, but does have very low memory needs.
* convolution will work.
*
* In #VIPS_ACCESS_RANDOM mode, small images are decompressed to memory and
* then processed from there. Large images are decompressed to temporary

23
libvips/iofuncs/object.c
View File

@ -1816,7 +1816,6 @@ vips_object_set_argument_from_string( VipsObject *object,
if( g_type_is_a( otype, VIPS_TYPE_IMAGE ) ) {
VipsImage *out;
VipsOperationFlags flags;
VipsAccess access;
if( !value ) {
vips_object_no_value( object, name );
@ -1830,15 +1829,10 @@ vips_object_set_argument_from_string( VipsObject *object,
/* Read the filename.
*/
if( flags & VIPS_OPERATION_SEQUENTIAL_UNBUFFERED )
access = VIPS_ACCESS_SEQUENTIAL_UNBUFFERED;
else if( flags & VIPS_OPERATION_SEQUENTIAL )
access = VIPS_ACCESS_SEQUENTIAL;
else
access = VIPS_ACCESS_RANDOM;
if( !(out = vips_image_new_from_file( value,
"access", access,
"access", flags & VIPS_OPERATION_SEQUENTIAL ?
VIPS_ACCESS_SEQUENTIAL : VIPS_ACCESS_RANDOM,
NULL )) )
return( -1 );
@ -1857,7 +1851,6 @@ vips_object_set_argument_from_string( VipsObject *object,
*/
VipsArrayImage *array_image;
VipsOperationFlags flags;
VipsAccess access;
if( !value ) {
vips_object_no_value( object, name );
@ -1869,15 +1862,9 @@ vips_object_set_argument_from_string( VipsObject *object,
flags = vips_operation_get_flags(
VIPS_OPERATION( object ) );
if( flags & VIPS_OPERATION_SEQUENTIAL_UNBUFFERED )
access = VIPS_ACCESS_SEQUENTIAL_UNBUFFERED;
else if( flags & VIPS_OPERATION_SEQUENTIAL )
access = VIPS_ACCESS_SEQUENTIAL;
else
access = VIPS_ACCESS_RANDOM;
if( !(array_image =
vips_array_image_new_from_string( value, access )) )
if( !(array_image = vips_array_image_new_from_string( value,
flags & VIPS_OPERATION_SEQUENTIAL ?
VIPS_ACCESS_SEQUENTIAL : VIPS_ACCESS_RANDOM )) )
return( -1 );
g_value_init( &gvalue, VIPS_TYPE_ARRAY_IMAGE );

15
libvips/iofuncs/operation.c
View File

@ -169,7 +169,6 @@
* VipsOperationFlags:
* @VIPS_OPERATION_NONE: no flags
* @VIPS_OPERATION_SEQUENTIAL: can work sequentially with a small buffer
* @VIPS_OPERATION_SEQUENTIAL_UNBUFFERED: can work sequentially with no buffer
* @VIPS_OPERATION_NOCACHE: must not be cached
* @VIPS_OPERATION_DEPRECATED: a compatibility thing
*
@ -179,20 +178,6 @@
* it can process images top-to-bottom with only small non-local
* references.
*
* Every scan-line must be requested, you are not allowed to skip
* ahead, but as a special case, the very first request can be for a region
* not at the top of the image. In this case, the first part of the image will
* be read and discarded
*
* @VIPS_OPERATION_SEQUENTIAL_UNBUFFERED means that the operation works like
* vips_copy(): it can process images top-to-bottom and makes no
* non-local references.
*
* Every scan-line must be requested, you are not allowed to skip
* ahead, but as a special case, the very first request can be for a region
* not at the top of the image. In this case, the first part of the image will
* be read and discarded
*
* @VIPS_OPERATION_NOCACHE means that the operation must not be cached by
* vips.
*

3
libvips/resample/shrink.c
View File

@ -128,9 +128,6 @@ vips_shrink_class_init( VipsShrinkClass *class )
vobject_class->description = _( "shrink an image" );
vobject_class->build = vips_shrink_build;
/* You'd think UNBUFFERED would work, but we will use reduce for non-int
* shrinks, so it has to be straight SEQ.
*/
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_DOUBLE( class, "vshrink", 9,

2
libvips/resample/shrinkh.c
View File

@ -320,7 +320,7 @@ vips_shrinkh_class_init( VipsShrinkhClass *class )
vobject_class->description = _( "shrink an image horizontally" );
vobject_class->build = vips_shrinkh_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_INT( class, "hshrink", 8,
_( "Hshrink" ),

2
libvips/resample/shrinkv.c
View File

@ -424,7 +424,7 @@ vips_shrinkv_class_init( VipsShrinkvClass *class )
vobject_class->description = _( "shrink an image vertically" );
vobject_class->build = vips_shrinkv_build;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL_UNBUFFERED;
operation_class->flags = VIPS_OPERATION_SEQUENTIAL;
VIPS_ARG_INT( class, "vshrink", 9,
_( "Vshrink" ),