Merge branch 'add-copy-to-memory'

ChangeLog

@@ -21,6 +21,7 @@
 - small tiffsave doc improvements
 - better thresholding for tiffsave "squash" mode
 - add @miniswhite mode to tiffsave
+- add vips_image_copy_memory(), improve stability with heavy threading
 
 6/2/15 started 7.42.3
 - bump version for back-compat ABI change

libvips/conversion/rot45.c

@@ -184,9 +184,9 @@ vips_rot45_build( VipsObject *object )
 	VipsObjectClass *class = VIPS_OBJECT_GET_CLASS( object );
 	VipsConversion *conversion = VIPS_CONVERSION( object );
 	VipsRot45 *rot45 = (VipsRot45 *) object;
-	VipsImage **t = (VipsImage **) vips_object_local_array( object, 1 );
+	VipsImage **t = (VipsImage **) vips_object_local_array( object, 2 );
 
-	VipsImage *from;
+	VipsImage *in;
 
 	if( VIPS_OBJECT_CLASS( vips_rot45_parent_class )->build( object ) )
 		return( -1 );
@@ -197,8 +197,9 @@ vips_rot45_build( VipsObject *object )
 	if( rot45->angle == VIPS_ANGLE45_D0 )
 		return( vips_image_write( rot45->in, conversion->out ) );
 
-	if( vips_image_wio_input( rot45->in ) )
+	if( !(t[1] = vips_image_copy_memory( rot45->in )) )
 		return( -1 );
+	in = t[1];
 
 	t[0] = vips_image_new_memory();
 	if( vips_image_pipelinev( t[0], 
@@ -207,35 +208,34 @@ vips_rot45_build( VipsObject *object )
 	if( vips_image_write_prepare( t[0] ) )
 		return( -1 );
 
-	from = rot45->in;
-
 	switch( rot45->angle ) {
 	case VIPS_ANGLE45_D315:
-		vips_rot45_rot45( t[0], from );
+		vips_rot45_rot45( t[0], in );
-		from = t[0];
+		in = t[0];
 		
 	case VIPS_ANGLE45_D270:
-		vips_rot45_rot45( t[0], from );
+		vips_rot45_rot45( t[0], in );
-		from = t[0];
+		in = t[0];
 
 	case VIPS_ANGLE45_D225:
-		vips_rot45_rot45( t[0], from );
+		vips_rot45_rot45( t[0], in );
-		from = t[0];
+		in = t[0];
 
 	case VIPS_ANGLE45_D180:
-		vips_rot45_rot45( t[0], from );
+		vips_rot45_rot45( t[0], in );
-		from = t[0];
+		in = t[0];
 
 	case VIPS_ANGLE45_D135:
-		vips_rot45_rot45( t[0], from );
+		vips_rot45_rot45( t[0], in );
-		from = t[0];
+		in = t[0];
 
 	case VIPS_ANGLE45_D90:
-		vips_rot45_rot45( t[0], from );
+		vips_rot45_rot45( t[0], in );
-		from = t[0];
+		in = t[0];
 
 	case VIPS_ANGLE45_D45:
-		vips_rot45_rot45( t[0], from );
+		vips_rot45_rot45( t[0], in );
+		in = t[0];
 		break;
 
 	default:
@@ -246,7 +246,7 @@ vips_rot45_build( VipsObject *object )
 		return( 0 );
 	}
 
-	if( vips_image_write( t[0], conversion->out ) )
+	if( vips_image_write( in, conversion->out ) )
 		return( -1 );
 
 	return( 0 );

libvips/convolution/correlation.c

@@ -81,7 +81,7 @@ vips_correlation_build( VipsObject *object )
 	VipsObjectClass *class = VIPS_OBJECT_GET_CLASS( object );
 	VipsCorrelationClass *cclass = VIPS_CORRELATION_CLASS( class );
 	VipsCorrelation *correlation = (VipsCorrelation *) object;
-	VipsImage **t = (VipsImage **) vips_object_local_array( object, 5 );
+	VipsImage **t = (VipsImage **) vips_object_local_array( object, 6 );
 
 	if( VIPS_OBJECT_CLASS( vips_correlation_parent_class )->
 		build( object ) )
@@ -98,11 +98,11 @@ vips_correlation_build( VipsObject *object )
 		return( -1 );
 	if( vips__formatalike( t[0], correlation->ref, &t[1], &t[2] ) ||
 		vips__bandalike( class->nickname, t[1], t[2], &t[3], &t[4] ) ||
-		vips_image_wio_input( t[4] ) ) 
+		!(t[5] = vips_image_copy_memory( t[4] )) )
 		return( -1 );
 
 	correlation->in_ready = t[3];
-	correlation->ref_ready = t[4];
+	correlation->ref_ready = t[5];
 
 	g_object_set( object, "out", vips_image_new(), NULL ); 
 

libvips/foreign/csv.c

@@ -430,19 +430,25 @@ int
 vips__csv_write( VipsImage *in, const char *filename, const char *separator )
 {
 	FILE *fp;
+	VipsImage *memory;
 
 	if( vips_check_mono( "vips2csv", in ) ||
 		vips_check_uncoded( "vips2csv", in ) ||
-		vips_image_wio_input( in ) )
+		!(memory = vips_image_copy_memory( in )) )
 		return( -1 );
 
-	if( !(fp = vips__file_open_write( filename, TRUE )) ) 
+	if( !(fp = vips__file_open_write( filename, TRUE )) ) { 
+		VIPS_UNREF( memory ); 
 		return( -1 );
-	if( vips2csv( in, fp, separator ) ) {
+	}
+
+	if( vips2csv( memory, fp, separator ) ) {
 		fclose( fp );
+		VIPS_UNREF( memory ); 
 		return( -1 );
 	}
 	fclose( fp );
+	VIPS_UNREF( memory ); 
 
 	return( 0 );
 }

libvips/foreign/vips2webp.c

@@ -62,11 +62,12 @@ int
 vips__webp_write_file( VipsImage *in, const char *filename, 
 	int Q, gboolean lossless )
 {
+	VipsImage *memory;
 	size_t len;
 	uint8_t *buffer;
 	FILE *fp;
 
-	if( vips_image_wio_input( in ) )
+	if( !(memory = vips_image_copy_memory( in )) )
 		return( -1 );
 
 	if( lossless ) {
@@ -77,10 +78,11 @@ vips__webp_write_file( VipsImage *in, const char *filename,
 		else
 			encoder = WebPEncodeLosslessRGB;
 
-		if( !(len = encoder( VIPS_IMAGE_ADDR( in, 0, 0 ), 
+		if( !(len = encoder( VIPS_IMAGE_ADDR( memory, 0, 0 ), 
-			in->Xsize, in->Ysize, 
+			memory->Xsize, memory->Ysize, 
-			VIPS_IMAGE_SIZEOF_LINE( in ),
+			VIPS_IMAGE_SIZEOF_LINE( memory ),
 			&buffer )) ) {
+			VIPS_UNREF( memory ); 
 			vips_error( "vips2webp", 
 				"%s", _( "unable to encode" ) ); 
 			return( -1 );
@@ -94,16 +96,19 @@ vips__webp_write_file( VipsImage *in, const char *filename,
 		else
 			encoder = WebPEncodeRGB;
 
-		if( !(len = encoder( VIPS_IMAGE_ADDR( in, 0, 0 ), 
+		if( !(len = encoder( VIPS_IMAGE_ADDR( memory, 0, 0 ), 
-			in->Xsize, in->Ysize, 
+			memory->Xsize, memory->Ysize, 
-			VIPS_IMAGE_SIZEOF_LINE( in ),
+			VIPS_IMAGE_SIZEOF_LINE( memory ),
 			Q, &buffer )) ) {
+			VIPS_UNREF( memory ); 
 			vips_error( "vips2webp", 
 				"%s", _( "unable to encode" ) ); 
 			return( -1 );
 		}
 	}
 
+	VIPS_UNREF( memory ); 
+
 	if( !(fp = vips__file_open_write( filename, FALSE )) ) {
 		free( buffer );
 		return( -1 );
@@ -125,9 +130,10 @@ int
 vips__webp_write_buffer( VipsImage *in, void **obuf, size_t *olen, 
 	int Q, gboolean lossless )
 {
+	VipsImage *memory;
 	webp_encoder encoder;
 
-	if( vips_image_wio_input( in ) )
+	if( !(memory = vips_image_copy_memory( in )) )
 		return( -1 );
 
 	if( in->Bands == 4 )
@@ -135,13 +141,15 @@ vips__webp_write_buffer( VipsImage *in, void **obuf, size_t *olen,
 	else
 		encoder = WebPEncodeRGB;
 
-	if( !(*olen = encoder( VIPS_IMAGE_ADDR( in, 0, 0 ), 
+	if( !(*olen = encoder( VIPS_IMAGE_ADDR( memory, 0, 0 ), 
-		in->Xsize, in->Ysize, 
+		memory->Xsize, memory->Ysize, 
-		VIPS_IMAGE_SIZEOF_LINE( in ),
+		VIPS_IMAGE_SIZEOF_LINE( memory ),
 		Q, (uint8_t **) obuf )) ) {
+		VIPS_UNREF( memory );
 		vips_error( "vips2webp", "%s", _( "unable to encode" ) ); 
 		return( -1 );
 	}
+	VIPS_UNREF( memory );
 
 	return( 0 );
 }

libvips/foreign/vipspng.c

@@ -707,6 +707,7 @@ const char *vips__png_suffs[] = { ".png", NULL };
  */
 typedef struct {
 	VipsImage *in;
+	VipsImage *memory;
 
 	FILE *fp;
 	png_structp pPng;
@@ -718,6 +719,7 @@ static void
 write_finish( Write *write )
 {
 	VIPS_FREEF( fclose, write->fp );
+	VIPS_UNREF( write->memory );
 	if( write->pPng )
 		png_destroy_write_struct( &write->pPng, &write->pInfo );
 }
@@ -737,6 +739,7 @@ write_new( VipsImage *in )
 		return( NULL );
 	memset( write, 0, sizeof( Write ) );
 	write->in = in;
+	write->memory = NULL;
 	g_signal_connect( in, "close", 
 		G_CALLBACK( write_destroy ), write ); 
 
@@ -820,8 +823,9 @@ write_vips( Write *write, int compress, int interlace, const char *profile,
 	 * we only suck once from upstream, switch to WIO. 
 	 */
 	if( interlace ) {
-		if( vips_image_wio_input( in ) )
+		if( !(write->memory = vips_image_copy_memory( in )) )
 			return( -1 );
+		in = write->memory;
 	}
 	else {
 		if( vips_image_pio_input( in ) )

libvips/histogram/histogram.c

@@ -118,6 +118,7 @@ vips_histogram_build( VipsObject *object )
 	VipsImage **format;
 	VipsImage **band;
 	VipsImage **size;
+	VipsImage **memory;
 
 	VipsPel *outbuf;		
 	VipsPel **inbuf;		
@@ -142,6 +143,7 @@ vips_histogram_build( VipsObject *object )
 	format = (VipsImage **) vips_object_local_array( object, histogram->n );
 	band = (VipsImage **) vips_object_local_array( object, histogram->n );
 	size = (VipsImage **) vips_object_local_array( object, histogram->n );
+	memory = (VipsImage **) vips_object_local_array( object, histogram->n );
 
 	g_object_set( histogram, "out", vips_image_new(), NULL ); 
 
@@ -169,14 +171,25 @@ vips_histogram_build( VipsObject *object )
 		vips__hist_sizealike_vec( band, size, histogram->n ) ) 
 		return( -1 );
 
-	/* Keep a copy of the processed images here for subclasses.
-	 */
-	histogram->ready = size;
-
 	if( vips_image_pipeline_array( histogram->out, 
-		VIPS_DEMAND_STYLE_THINSTRIP, histogram->ready ) ) 
+		VIPS_DEMAND_STYLE_THINSTRIP, size ) ) 
 		return( -1 );
 
+	/* Need a copy of the inputs in memory.
+	 */
+	if( !(inbuf = VIPS_ARRAY( object, histogram->n + 1, VipsPel * )) )
+                return( -1 );
+	for( i = 0; i < histogram->n; i++ ) {
+		if( !(memory[i] = vips_image_copy_memory( size[i] )) )
+			return( -1 ); 
+		inbuf[i] = VIPS_IMAGE_ADDR( memory[i], 0, 0 );
+	}
+	inbuf[i] = NULL; 
+
+	/* Keep a copy of the memory images here for subclasses.
+	 */
+	histogram->ready = memory;
+
 	histogram->out->Xsize = VIPS_IMAGE_N_PELS( histogram->ready[0] );
 	histogram->out->Ysize = 1;
 	if( hclass->format_table ) 
@@ -188,15 +201,6 @@ vips_histogram_build( VipsObject *object )
 		VIPS_IMAGE_SIZEOF_LINE( histogram->out ))) )
                 return( -1 );
 
-	if( !(inbuf = VIPS_ARRAY( object, histogram->n + 1, VipsPel * )) )
-                return( -1 );
-	for( i = 0; i < histogram->n; i++ ) {
-		if( vips_image_wio_input( histogram->ready[i] ) )
-			return( -1 ); 
-		inbuf[i] = VIPS_IMAGE_ADDR( histogram->ready[i], 0, 0 );
-	}
-	inbuf[i] = NULL; 
-
 	hclass->process( histogram, outbuf, inbuf, histogram->ready[0]->Xsize );
 
 	if( vips_image_write_line( histogram->out, 0, outbuf ) )

libvips/histogram/maplut.c

@@ -561,7 +561,7 @@ vips_maplut_build( VipsObject *object )
 {
 	VipsObjectClass *class = VIPS_OBJECT_GET_CLASS( object );
 	VipsMaplut *maplut = (VipsMaplut *) object;
-	VipsImage **t = (VipsImage **) vips_object_local_array( object, 1 );
+	VipsImage **t = (VipsImage **) vips_object_local_array( object, 2 );
 
 	VipsImage *in;
 	VipsImage *lut;
@@ -576,8 +576,7 @@ vips_maplut_build( VipsObject *object )
 	lut = maplut->lut;
 
 	if( vips_check_hist( class->nickname, lut ) ||
-		vips_check_uncoded( class->nickname, lut ) ||
+		vips_check_uncoded( class->nickname, lut ) )
-		vips_image_wio_input( lut ) )
 		return( -1 );
 
 	/* Cast @in to u8/u16/u32 to make the index image.
@@ -617,6 +616,9 @@ vips_maplut_build( VipsObject *object )
 	/* Make luts. We unpack the LUT image into a 2D C array to speed
 	 * processing.
 	 */
+	if( !(t[1] = vips_image_copy_memory( lut )) )
+		return( -1 );
+	lut = t[1];
 	maplut->fmt = lut->BandFmt;
 	maplut->es = VIPS_IMAGE_SIZEOF_ELEMENT( lut );
 	maplut->sz = lut->Xsize * lut->Ysize;

libvips/include/vips/image.h

@@ -458,6 +458,7 @@ gboolean vips_image_isMSBfirst( VipsImage *image );
 gboolean vips_image_isfile( VipsImage *image );
 gboolean vips_image_ispartial( VipsImage *image );
 
+VipsImage *vips_image_copy_memory( VipsImage *image );
 int vips_image_wio_input( VipsImage *image );
 int vips_image_pio_input( VipsImage *image );
 int vips_image_pio_output( VipsImage *image );

libvips/iofuncs/error.c

@@ -1282,6 +1282,8 @@ vips_check_hist( const char *domain, VipsImage *im )
 int
 vips_check_matrix( const char *domain, VipsImage *im, VipsImage **out )
 {
+	VipsImage *t;
+
 	*out = NULL;
 
 	if( im->Xsize > 100000 || 
@@ -1295,10 +1297,13 @@ vips_check_matrix( const char *domain, VipsImage *im, VipsImage **out )
 		return( -1 );
 	}
 
-	if( vips_cast( im, out, VIPS_FORMAT_DOUBLE, NULL ) )
+	if( vips_cast( im, &t, VIPS_FORMAT_DOUBLE, NULL ) )
                 return( -1 );
-        if( vips_image_wio_input( *out ) )
+	if( !(*out = vips_image_copy_memory( t )) ) {
+		VIPS_UNREF( t );
 		return( -1 );
+	}
+	VIPS_UNREF( t );
 
 	return( 0 );
 }

libvips/iofuncs/generate.c

@@ -742,6 +742,11 @@ vips_image_generate( VipsImage *image,
                 if( res )
                         return( -1 );
 
+		/* We've written to image ... rewind it ready for reading.
+		 */
+		if( vips_image_pio_input( image ) )
+			return( -1 ); 
+
                 break;
  
         default:

libvips/iofuncs/header.c

@@ -654,7 +654,10 @@ vips_image_get_offset( const VipsImage *image )
  * allocating large amounts of memory and performing a long computation. Image
  * pixels are laid out in band-packed rows.
  *
- * See also: vips_image_wio_input().
+ * Since this function modifies @image, it is not threadsafe. Only call it on
+ * images which you are sure have not been shared with another thread. 
+ *
+ * See also: vips_image_wio_input(), vips_image_copy_memory().
  *
  * Returns: (transfer none): a pointer to pixel data, if possible.
  */

libvips/iofuncs/image.c

@@ -4,6 +4,8 @@
  * 	- hacked up from various places
  * 6/6/13
  * 	- vips_image_write() didn't ref non-partial sources
+ * 18/4/15
+ * 	- add vips_image_copy_memory()
  */
 
 /*
@@ -2842,8 +2844,8 @@ vips_image_rewind_output( VipsImage *image )
 
 	/* And reopen ... recurse to get a mmaped image. 
 	 *
-	 * We use "v" mode to get it opened as a vips image, byopassing the
+	 * We use "v" mode to get it opened as a vips image, bypassing the
-	 * file type checks. They will fail on Windows becasue you can't open
+	 * file type checks. They will fail on Windows because you can't open
 	 * fds more than once.
 	 */
 	image->fd = fd;
@@ -2873,18 +2875,74 @@ vips_image_rewind_output( VipsImage *image )
 	return( 0 );
 }
 
+/** 
+ * vips_image_copy_memory:
+ * @image: image to copy to a memory buffer
+ *
+ * Allocate a memory buffer and copy @image to it. This is a thread-safe
+ * equivalent of vips_image_wio_input(), useful if @image is small and from an
+ * unknown source.
+ *
+ * If @image is already in memory (perhaps a mmaped file on disc),
+ * vips_image_copy_memory() will just ref @image and return that.
+ *
+ * If you are sure that @image is not shared with another thread (perhaps you
+ * have made it yourself), use vips_image_wio_input() instead.
+ *
+ * See also: vips_image_wio_input().
+ */
+VipsImage *
+vips_image_copy_memory( VipsImage *image )
+{
+	VipsImage *new;
+
+	switch( image->dtype ) {
+	case VIPS_IMAGE_SETBUF:
+	case VIPS_IMAGE_SETBUF_FOREIGN:
+	case VIPS_IMAGE_MMAPIN:
+	case VIPS_IMAGE_MMAPINRW:
+		/* Can read from all these, in principle anyway.
+		 */
+		new = image;
+		g_object_ref( new );
+		break;
+
+	case VIPS_IMAGE_OPENOUT:
+	case VIPS_IMAGE_OPENIN:
+	case VIPS_IMAGE_PARTIAL:
+		/* Copy to a new memory image.
+		 */
+		new = vips_image_new_memory();
+		if( vips_image_write( image, new ) ) {
+			g_object_unref( new );
+			return( NULL );
+		}
+		break;
+
+	default:
+		vips_error( "vips_image_copy_memory", 
+			"%s", _( "image not readable" ) );
+		return( NULL );
+	}
+
+	return( new );
+}
+
 /**
  * vips_image_wio_input:
  * @image: image to transform
  *
  * Check that an image is readable via the VIPS_IMAGE_ADDR() macro, that is,
  * that the entire image is in memory and all pixels can be read with 
- * VIPS_IMAGE_ADDR().
+ * VIPS_IMAGE_ADDR().  If it 
- *
- * If it 
  * isn't, try to transform it so that VIPS_IMAGE_ADDR() can work. 
  *
- * See also: vips_image_pio_input(), vips_image_inplace(), VIPS_IMAGE_ADDR().
+ * Since this function modifies @image, it is not thread-safe. Only call it on
+ * images which you are sure have not been shared with another thread. If the
+ * image might have been shared, use the less efficient
+ * vips_image_copy_memory() instead.
+ *
+ * See also: vips_image_copy_memory(), vips_image_pio_input(), vips_image_inplace(), VIPS_IMAGE_ADDR().
  *
  * Returns: 0 on succeess, or -1 on error.
  */
@@ -3053,6 +3111,11 @@ vips__image_wio_output( VipsImage *image )
  * After calling this function you can both read and write the image with 
  * VIPS_IMAGE_ADDR().
  *
+ * Since this function modifies @image, it is not thread-safe. Only call it on
+ * images which you are sure have not been shared with another thread. 
+ * All in-place operations are inherently not thread-safe, so you need to take
+ * great care in any case.
+ *
  * See also: vips_draw_circle(), vips_image_wio_input().
  *
  * Returns: 0 on succeess, or -1 on error.

libvips/resample/quadratic.c

@@ -311,13 +311,14 @@ vips_quadratic_build( VipsObject *object )
 		NULL ) )
 		return( -1 );
 	vips_object_local( object, t );
-
 	in = t;
 
         /* We need random access to our input.
          */
-        if( vips_image_wio_input( in ) )
+        if( !(t = vips_image_copy_memory( in )) )
                 return( -1 );
+	vips_object_local( object, t );
+	in = t;
 
 	if( vips_image_generate( resample->out,
 		vips_start_one, vips_quadratic_gen, vips_stop_one,