fix up new seq mode stuff

seems to work

TODO

@@ -11,38 +11,9 @@
  
 - resize.c has
 
-                need_lines = 1.2 * n_lines / vscale;
+- VIPS_META_SEQUENTIAL will be saved to vips files, then loaded back again argh
 
-  to size the line cache, and uses an unthreaded cache
+  we need to set a value that can't be saved
-
-  	- is this formula right?
-	- instead put the cache inside shrinkv, on the output
-	- make shrinkv use threads to generate lines for input?
-
-  another alternative
-
-  	- add this caching to thumbnail.c
-	- maybe remove the cache from resize? does sharp need it? nope, safe to
-	  remove
-	- don't use vips_resize(), expand code out
-	- put the cache just after shrinkv
-	- now there's no need to thread, since thumbnail is (mostly) just 
-	  doing image load on the input to shrink
-	- and we leave shink and resize nice and simple, though they will no
-	  longer work on seq files
-
-	- should linecache ask for inbetween lines in unthreaded seq mode?
-	- could allow many readers, but only one writer active at once? perhaps
-	  that's what we have?
-
-	- sharp makes quite a few assumptions about cache size, it'd be easy to
-	  break
-
-	- loaders could add a "seq-mode" meta tag, shrinkv could single-thread
-	  if it sees the tag
-	- the seq tag could just be added by vips_sequential
-	- good idea! keeps compat with everything, keeps non-seq vips siomple
-	  and quick
 
   do we need a seq cache for reducev as well? we could have up to a 3x
   reduction there

libvips/conversion/sequential.c

@@ -299,7 +299,6 @@ vips_sequential_build( VipsObject *object )
 	if( vips_image_pipelinev( conversion->out, 
 		VIPS_DEMAND_STYLE_THINSTRIP, t, NULL ) )
 		return( -1 );
-	vips_image_set_int( conversion->out, VIPS_META_SEQUENTIAL, 1 ); 
 	if( vips_image_generate( conversion->out,
 		vips_start_one, vips_sequential_generate, vips_stop_one, 
 		t, sequential ) )

libvips/foreign/jpeg2vips.c

@@ -514,6 +514,10 @@ read_jpeg_header( ReadJpeg *jpeg, VipsImage *out )
 	if( vips__exif_parse( out ) )
 		return( -1 );
 
+	/* Tell downstream we are reading sequentially.
+	 */
+	vips_image_set_int( out, VIPS_META_SEQUENTIAL, 1 ); 
+
 	return( 0 );
 }
 

libvips/foreign/radiance.c

@@ -1082,6 +1082,10 @@ rad2vips_get_header( Read *read, VipsImage *out )
 			vips_image_set_double( out, 
 				prims_name[i][j], read->prims[i][j] );
 
+	/* Tell downstream we are reading sequentially.
+	 */
+	vips_image_set_int( out, VIPS_META_SEQUENTIAL, 1 ); 
+
 	return( 0 );
 }
 

libvips/foreign/tiff2vips.c

@@ -1334,6 +1334,11 @@ rtiff_set_header( Rtiff *rtiff, VipsImage *out )
 	vips_image_set_int( out, 
 		VIPS_META_ORIENTATION, rtiff->header.orientation );
 
+	/* Tell downstream if we are reading sequentially.
+	 */
+	if( !rtiff->header.tiled ) 
+		vips_image_set_int( out, VIPS_META_SEQUENTIAL, 1 ); 
+
 	return( 0 );
 }
 

libvips/foreign/vipspng.c

@@ -382,10 +382,19 @@ png2vips_header( Read *read, VipsImage *out )
 		VIPS_CODING_NONE, interpretation, 
 		Xres, Yres );
 
-	/* Sequential mode needs thinstrip to work with things like
+	/* Uninterlaced images will be read in seq mode. Interlaced images are
-	 * vips_shrink().
+	 * read via a huge memory buffer.
 	 */
-        vips_image_pipelinev( out, VIPS_DEMAND_STYLE_THINSTRIP, NULL );
+	if( interlace_type == PNG_INTERLACE_NONE ) {
+		vips_image_set_int( out, VIPS_META_SEQUENTIAL, 1 ); 
+
+		/* Sequential mode needs thinstrip to work with things like
+		 * vips_shrink().
+		 */
+		vips_image_pipelinev( out, VIPS_DEMAND_STYLE_THINSTRIP, NULL );
+	}
+	else 
+		vips_image_pipelinev( out, VIPS_DEMAND_STYLE_ANY, NULL );
 
 	/* Fetch the ICC profile. @name is useless, something like "icc" or
 	 * "ICC Profile" etc.  Ignore it.

libvips/resample/shrinkv.c

@@ -272,13 +272,6 @@ vips_shrinkv_gen( VipsRegion *or, void *vseq,
 	 * the input region corresponding to *r since it could be huge. 
 	 *
 	 * Request input a line at a time, average to a line buffer.  
-	 *
-	 * We don't chunk horizontally. We want "vips shrink x.jpg b.jpg 100
-	 * 100" to run sequentially. If we chunk horizontally, we will fetch
-	 * 100x100 lines from the top of the image, then 100x100 100 lines
-	 * down, etc. for each thread, then when they've finished, fetch
-	 * 100x100, 100 pixels across from the top of the image. This will
-	 * break sequentiality. 
 	 */
 
 #ifdef DEBUG
@@ -330,7 +323,7 @@ vips_shrinkv_build( VipsObject *object )
 	VipsResample *resample = VIPS_RESAMPLE( object );
 	VipsShrinkv *shrink = (VipsShrinkv *) object;
 	VipsImage **t = (VipsImage **) 
-		vips_object_local_array( object, 3 );
+		vips_object_local_array( object, 4 );
 
 	VipsImage *in;
 
@@ -365,6 +358,48 @@ vips_shrinkv_build( VipsObject *object )
 		return( -1 );
 	in = t[1];
 
+	/* We have to keep a line buffer as we sum columns.
+	 */
+	shrink->sizeof_line_buffer = 
+		in->Xsize * in->Bands * 
+		vips_format_sizeof( VIPS_FORMAT_DPCOMPLEX );
+
+	/* SMALLTILE or we'll need huge input areas for our output. In seq
+	 * mode, the linecache above will keep us sequential. 
+	 */
+	t[2] = vips_image_new();
+	if( vips_image_pipelinev( t[2],
+		VIPS_DEMAND_STYLE_SMALLTILE, in, NULL ) )
+		return( -1 );
+
+	/* Size output. We need to always round to nearest, so round(), not
+	 * rint().
+	 *
+	 * Don't change xres/yres, leave that to the application layer. For
+	 * example, vipsthumbnail knows the true shrink factor (including the
+	 * fractional part), we just see the integer part here.
+	 */
+	t[2]->Ysize = VIPS_ROUND_UINT( 
+		(double) resample->in->Ysize / shrink->vshrink );
+	if( t[2]->Ysize <= 0 ) {
+		vips_error( class->nickname, 
+			"%s", _( "image has shrunk to nothing" ) );
+		return( -1 );
+	}
+
+#ifdef DEBUG
+	printf( "vips_shrinkv_build: shrinking %d x %d image to %d x %d\n", 
+		in->Xsize, in->Ysize, 
+		t[2]->Xsize, t[2]->Ysize );  
+#endif /*DEBUG*/
+
+	if( vips_image_generate( t[2],
+		vips_shrinkv_start, vips_shrinkv_gen, vips_shrinkv_stop, 
+		in, shrink ) )
+		return( -1 );
+
+	in = t[2];
+
 	/* Large vshrinks will throw off sequential mode. Suppose thread1 is
 	 * generating tile (0, 0), but stalls. thread2 generates tile
 	 * (0, 1), 128 lines further down the output. After it has done,
@@ -386,9 +421,10 @@ vips_shrinkv_build( VipsObject *object )
 		int tile_height;
 		int n_lines;
 
+		g_info( "shrinkv sequential line cache" ); 
 		vips_get_tile_size( in, 
 			&tile_width, &tile_height, &n_lines );
-		if( vips_tilecache( in, &t[2], 
+		if( vips_tilecache( in, &t[3], 
 			"tile_width", in->Xsize,
 			"tile_height", 10,
 			"max_tiles", 1 + n_lines / 10,
@@ -396,47 +432,10 @@ vips_shrinkv_build( VipsObject *object )
 			"threaded", FALSE, 
 			NULL ) )
 			return( -1 );
-		in = t[2];
+		in = t[3];
 	}
 
-	/* We have to keep a line buffer as we sum columns.
+	if( vips_image_write( in, resample->out ) )
-	 */
-	shrink->sizeof_line_buffer = 
-		in->Xsize * in->Bands * 
-		vips_format_sizeof( VIPS_FORMAT_DPCOMPLEX );
-
-	/* THINSTRIP will work, anything else will break seq mode. If you 
-	 * combine shrink with conv you'll need to use a line cache to maintain
-	 * sequentiality.
-	 */
-	if( vips_image_pipelinev( resample->out, 
-		VIPS_DEMAND_STYLE_THINSTRIP, in, NULL ) )
-		return( -1 );
-
-	/* Size output. We need to always round to nearest, so round(), not
-	 * rint().
-	 *
-	 * Don't change xres/yres, leave that to the application layer. For
-	 * example, vipsthumbnail knows the true shrink factor (including the
-	 * fractional part), we just see the integer part here.
-	 */
-	resample->out->Ysize = VIPS_ROUND_UINT( 
-		(double) resample->in->Ysize / shrink->vshrink );
-	if( resample->out->Ysize <= 0 ) {
-		vips_error( class->nickname, 
-			"%s", _( "image has shrunk to nothing" ) );
-		return( -1 );
-	}
-
-#ifdef DEBUG
-	printf( "vips_shrinkv_build: shrinking %d x %d image to %d x %d\n", 
-		in->Xsize, in->Ysize, 
-		resample->out->Xsize, resample->out->Ysize );  
-#endif /*DEBUG*/
-
-	if( vips_image_generate( resample->out,
-		vips_shrinkv_start, vips_shrinkv_gen, vips_shrinkv_stop, 
-		in, shrink ) )
 		return( -1 ); 
 
 	return( 0 );