Merge branch '8.5'

ChangeLog

@@ -1,6 +1,9 @@
 15/4/17 started 8.6.0
 - supports fits images with leading non-image HDUs, thanks benepo
 
+23/4/17 started 8.5.4
+- don't depend on image width when setting n_lines
+
 7/4/17 started 8.5.3
 - more link fixing in docs 
 - revise cache sizing again to help out of order errors under heavy load, thanks

configure.ac

@@ -38,7 +38,7 @@ VIPS_VERSION_STRING=$VIPS_VERSION-`date`
 #   binary interface changes not backwards compatible?: reset age to 0
 
 LIBRARY_CURRENT=49
-LIBRARY_REVISION=4
+LIBRARY_REVISION=5
 LIBRARY_AGE=7
 
 # patched into include/vips/version.h

libvips/conversion/sequential.c

@@ -117,8 +117,10 @@ vips_sequential_generate( VipsRegion *or,
         VipsRect *r = &or->valid;
 	VipsRegion *ir = (VipsRegion *) seq;
 
-	VIPS_DEBUG_MSG_GREEN( "thread %p request for line %d, height %d\n", 
-		g_thread_self(), r->top, r->height );
+	if( sequential->trace )
+		printf( "vips_sequential_generate %p: "
+			"request for line %d, height %d\n", 
+			sequential, r->top, r->height );
 
 	VIPS_GATE_START( "vips_sequential_generate: wait" );
 
@@ -141,9 +143,10 @@ vips_sequential_generate( VipsRegion *or,
 		 */
 		VipsRect area;
 
-		VIPS_DEBUG_MSG_GREEN( "thread %p skipping to line %d ...\n", 
-			g_thread_self(),
-			r->top );
+		if( sequential->trace )
+			printf( "vips_sequential_generate %p: "
+				"skipping to line %d ...\n", 
+				sequential, r->top );
 
 		area.left = 0;
 		area.top = sequential->y_pos;
@@ -236,7 +239,6 @@ vips_sequential_class_init( VipsSequentialClass *class )
 		G_STRUCT_OFFSET( VipsSequential, tile_height ),
 		1, 1000000, 1 );
 
-
 	VIPS_ARG_ENUM( class, "access", 6, 
 		_( "Strategy" ), 
 		_( "Expected access pattern" ),
@@ -259,6 +261,7 @@ vips_sequential_init( VipsSequential *sequential )
 	sequential->lock = vips_g_mutex_new();
 	sequential->tile_height = 1;
 	sequential->error = 0;
+	sequential->trace = FALSE;
 }
 
 /**

libvips/include/vips/threadpool.h

@@ -88,6 +88,11 @@ typedef struct _VipsThreadState {
 	 */
         void *a;
 
+	/* Set in allocate to stall this thread for a moment. Handy for
+	 * debugging race conditions.
+	 */
+	gboolean stall;
+
 } VipsThreadState;
 
 typedef struct _VipsThreadStateClass {

libvips/iofuncs/sinkdisc.c

@@ -331,6 +331,10 @@ wbuffer_allocate_fn( VipsThreadState *state, void *a, gboolean *stop )
 		sink_base->y += sink_base->tile_height;
 
 		if( sink_base->y >= VIPS_RECT_BOTTOM( &write->buf->area ) ) {
+			VIPS_DEBUG_MSG( "wbuffer_allocate_fn: "
+				"finished top = %d, height = %d\n",
+				write->buf->area.top, write->buf->area.height );
+
 			/* Block until the write of the previous buffer 
 			 * is done, then set write of this buffer going.
 			 */
@@ -346,10 +350,6 @@ wbuffer_allocate_fn( VipsThreadState *state, void *a, gboolean *stop )
 				return( 0 );
 			}
 
-			VIPS_DEBUG_MSG( "wbuffer_allocate_fn: "
-				"finished top = %d, height = %d\n",
-				write->buf->area.top, write->buf->area.height );
-
 			VIPS_DEBUG_MSG( "wbuffer_allocate_fn: "
 				"starting top = %d, height = %d\n",
 				sink_base->y, sink_base->nlines );
@@ -365,6 +365,11 @@ wbuffer_allocate_fn( VipsThreadState *state, void *a, gboolean *stop )
 				*stop = TRUE;
 				return( -1 );
 			}
+
+			/* This will be the first tile of a new buffer ...
+			 * stall for a moment to stress the caching system.
+			 */
+			state->stall = TRUE;
 		}
 	}
 

libvips/iofuncs/threadpool.c

@@ -18,6 +18,9 @@
  * 6/3/17
  * 	- remove single-thread-first-request thing, new seq system makes it
  * 	  unnecessary
+ * 23/4/17
+ * 	- add ->stall
+ * 	- don't depend on image width when setting n_lines
  */
 
 /*
@@ -111,6 +114,10 @@ int vips__n_active_threads = 0;
  */
 static GPrivate *is_worker_key = NULL;
 
+/* Set to stall threads for debugging.
+ */
+static gboolean vips__stall = FALSE;
+
 /* Glib 2.32 revised the thread API. We need some compat functions.
  */
 
@@ -466,6 +473,7 @@ vips_thread_state_init( VipsThreadState *state )
 
 	state->reg = NULL;
 	state->stop = FALSE;
+	state->stall = FALSE;
 }
 
 void *
@@ -634,6 +642,17 @@ vips_thread_work_unit( VipsThread *thr )
 
 	g_mutex_unlock( pool->allocate_lock );
 
+	if( thr->state->stall &&
+		vips__stall ) { 
+		/* Sleep for 0.5s. Handy for stressing the seq system. Stall
+		 * is set by allocate funcs in various places. 
+		 */
+		g_usleep( 500000 ); 
+		thr->state->stall = FALSE;
+		printf( "vips_thread_work_unit: "
+			"stall done, releasing y = %d ...\n", thr->state->y ); 
+	}
+
 	/* Process a work unit.
 	 */
 	if( pool->work( thr->state, pool->a ) ) { 
@@ -999,6 +1018,9 @@ vips__threadpool_init( void )
 	if( !is_worker_key ) 
 		is_worker_key = g_private_new( NULL ); 
 #endif
+
+	if( g_getenv( "VIPS_STALL" ) )
+		vips__stall = TRUE;
 }
 
 /**
@@ -1021,6 +1043,7 @@ vips_get_tile_size( VipsImage *im,
 	int *tile_width, int *tile_height, int *n_lines )
 {
 	const int nthr = vips_concurrency_get();
+	const int typical_image_width = 1000;
 
 	/* Compiler warnings.
 	 */
@@ -1054,11 +1077,15 @@ vips_get_tile_size( VipsImage *im,
 	 * the pipeline might see a different hint and we need to synchronise
 	 * buffer sizes everywhere.
 	 *
+	 * We also can't depend on the current image size, since that might
+	 * change down the pipeline too. Pick a typical image width.
+	 *
 	 * Pick the maximum buffer size we might possibly need, then round up
 	 * to a multiple of tileheight.
 	 */
 	*n_lines = vips__tile_height * 
-		VIPS_ROUND_UP( vips__tile_width * nthr, im->Xsize ) / im->Xsize;
+		VIPS_ROUND_UP( vips__tile_width * nthr, typical_image_width ) / 
+			typical_image_width;
 	*n_lines = VIPS_MAX( *n_lines, vips__fatstrip_height * nthr );
 	*n_lines = VIPS_MAX( *n_lines, vips__thinstrip_height * nthr );
 	*n_lines = VIPS_ROUND_UP( *n_lines, *tile_height );

libvips/resample/reducev.cpp

@@ -835,8 +835,6 @@ vips_reducev_build( VipsObject *object )
 	if( VIPS_OBJECT_CLASS( vips_reducev_parent_class )->build( object ) )
 		return( -1 );
 
-	g_info( "reducev by factor %g", reducev->vshrink );
-
 	in = resample->in; 
 
 	if( reducev->vshrink < 1 ) { 
@@ -895,6 +893,7 @@ vips_reducev_build( VipsObject *object )
 
 		if( vips_sequential( in, &t[3], 
 			"tile_height", 10,
+			// "trace", TRUE,
 			NULL ) )
 			return( -1 );
 		in = t[3];

test/test_python.sh

@@ -25,3 +25,7 @@ Darwin)
 esac
 
 $PYTHON -m unittest -v test_all 
+
+echo rerunning with VIPS_STALL enabled ...
+export VIPS_STALL=1
+$PYTHON -m unittest -v test_all 

test/test_thumbnail.sh

@@ -24,6 +24,9 @@ break_threshold() {
 	return $(echo "$diff > $threshold" | bc -l)
 }
 
+# run the test with VIPS_STALL enabled to stress the seq system
+export VIPS_STALL=1
+
 size=1000
 while [ $size -gt 99 ]; do
 	printf "testing size to $size ... "