ppm load uses streams

ChangeLog

@@ -17,8 +17,8 @@
 - nifti load/save uses double for all floating point metadata
 - add vips_error_buffer_copy()
 - add VipsStream: a universal IO class for loaders and savers
-- jpeg, png, tiff (though not tiffsave), rad, svg and webp use the new IO 
+- jpeg, png, tiff (though not tiffsave), rad, svg, ppm and webp use the 
-  class
+  new IO class
 - add @no_strip option to dzsave [kalozka1]
 - add iiif layout to dzsave
 - fix use of resolution-unit metadata on tiff save [kayarre]

libvips/foreign/pforeign.h

@@ -143,10 +143,10 @@ int vips__mat_load( const char *filename, VipsImage *out );
 int vips__mat_header( const char *filename, VipsImage *out );
 int vips__mat_ismat( const char *filename );
 
-int vips__ppm_header( const char *name, VipsImage *out );
+int vips__ppm_header_stream( VipsStreamib *streamib, VipsImage *out );
-int vips__ppm_load( const char *name, VipsImage *out );
+int vips__ppm_load_stream( VipsStreamib *streamib, VipsImage *out );
-int vips__ppm_isppm( const char *filename );
+VipsForeignFlags vips__ppm_flags_stream( VipsStreamib *streamib );
-VipsForeignFlags vips__ppm_flags( const char *filename );
+int vips__ppm_isppm_stream( VipsStreami *streami );
 extern const char *vips__ppm_suffs[];
 
 int vips__ppm_save_stream( VipsImage *in, VipsStreamo *streamo,

libvips/foreign/ppm.c

@@ -37,7 +37,7 @@
  * 29/7/19 Kyle-Kyle
  * 	- fix a loop with malformed ppm
  * 13/11/19
- * 	- ppm save redone with streams
+ * 	- redone with streams
  */
 
 /*
@@ -90,36 +90,78 @@
  */
 #define MAX_THING (80)
 
+/* After this, the next getc will be the first char of the next line (or EOF).
+ */
 static void 
-skip_line( FILE *fp )
+skip_line( VipsStreamib *streamib )
 {
 	int ch;
 
-        while( (ch = vips__fgetc( fp )) != '\n' && 
+        while( (ch = VIPS_STREAMIB_GETC( streamib )) != '\n' && 
 		ch != EOF )
 		;
 }
 
+/* After this, the next getc will be the first char of the next block of
+ * non-whitespace (or EOF).
+ */
 static void 
-skip_white_space( FILE *fp )
+skip_white_space( VipsStreamib *streamib )
 {
         int ch;
 
-        while( isspace( ch = vips__fgetc( fp ) ) )
+	while( isspace( ch = VIPS_STREAMIB_GETC( streamib ) ) )
 		;
-	ungetc( ch, fp );
+	VIPS_STREAMIB_UNGETC( streamib );
 
 	if( ch == '#' ) {
-		skip_line( fp );
+		skip_line( streamib );
-		skip_white_space( fp );
+		skip_white_space( streamib );
 	}
 }
 
-static int
+/* After this, the next getc will be the first char of the next block of
-read_int( FILE *fp, int *i )
+ * whitespace (or EOF). buf will be filled with up to length bytes, and
+ * null-terminated.
+ *
+ * If the first getc is whitespace, stop instantly and return nothing.
+ */
+static void
+read_non_white_space( VipsStreamib *streamib, char *buf, int length )
 {
-	skip_white_space( fp );
+	int ch;
-	if( fscanf( fp, "%d", i ) != 1 ) {
+	int i;
+
+	for( i = 0; i < length - 1 &&
+		!isspace( ch = VIPS_STREAMIB_GETC( streamib ) ) &&
+		ch != EOF; i++ ) 
+		buf[i] = ch;
+	buf[i] = '\0';
+
+	/* If we stopped before seeing any whitespace, skip to the end of the
+	 * block of non-whitespace.
+	 */
+	if( !isspace( ch ) ) 
+		while( !isspace( ch = VIPS_STREAMIB_GETC( streamib ) ) &&
+			ch != EOF )
+			;
+
+	/* If we finally stopped on whitespace, step back one so the next get
+	 * will be whitespace (or EOF).
+	 */
+	if( isspace( ch ) ) 
+		VIPS_STREAMIB_UNGETC( streamib );
+}
+
+static int
+read_int( VipsStreamib *streamib, int *i )
+{
+	char buf[MAX_THING];
+
+	skip_white_space( streamib );
+	read_non_white_space( streamib, buf, MAX_THING );
+
+	if( sscanf( buf, "%d", i ) != 1 ) {
 		vips_error( "ppm2vips", "%s", _( "bad int" ) );
 		return( -1 );
 	}
@@ -128,10 +170,14 @@ read_int( FILE *fp, int *i )
 }
 
 static int
-read_float( FILE *fp, float *f )
+read_float( VipsStreamib *streamib, float *f )
 {
-	skip_white_space( fp );
+	char buf[MAX_THING];
-	if( fscanf( fp, "%f", f ) != 1 ) {
+
+	skip_white_space( streamib );
+	read_non_white_space( streamib, buf, MAX_THING );
+
+	if( sscanf( buf, "%f", f ) != 1 ) {
 		vips_error( "ppm2vips", "%s", _( "bad float" ) );
 		return( -1 );
 	}
@@ -153,7 +199,8 @@ static char *magic_names[] = {
 };
 
 static int
-read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first )
+read_header( VipsStreamib *streamib, VipsImage *out, 
+	int *bits, int *ascii, int *msb_first )
 {
 	int width, height, bands; 
 	VipsBandFormat format;
@@ -175,8 +222,8 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first )
 
 	/* Read in the magic number.
 	 */
-	buf[0] = vips__fgetc( fp );
+	buf[0] = VIPS_STREAMIB_GETC( streamib );
-	buf[1] = vips__fgetc( fp );
+	buf[1] = VIPS_STREAMIB_GETC( streamib );
 	buf[2] = '\0';
 
 	for( index = 0; index < VIPS_NUMBER( magic_names ); index++ )
@@ -196,8 +243,8 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first )
 
 	/* Read in size.
 	 */
-	if( read_int( fp, &width ) ||
+	if( read_int( streamib, &width ) ||
-		read_int( fp, &height ) )
+		read_int( streamib, &height ) )
 		return( -1 );
 
 	/* Read in max value / scale for >1 bit images.
@@ -206,7 +253,7 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first )
 		if( index == 6 || index == 7 ) {
 			float scale;
 
-			if( read_float( fp, &scale ) )
+			if( read_float( streamib, &scale ) )
 				return( -1 );
 
 			/* Scale > 0 means big-endian.
@@ -218,7 +265,7 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first )
 		else {
 			int max_value;
 
-			if( read_int( fp, &max_value ) )
+			if( read_int( streamib, &max_value ) )
 				return( -1 );
 
 			if( max_value > 255 )
@@ -232,7 +279,7 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first )
 	 * character before the data starts.
 	 */
 	if( !*ascii && 
-		!isspace( vips__fgetc( fp ) ) ) {
+		!isspace( VIPS_STREAMIB_GETC( streamib ) ) ) {
 		vips_error( "ppm2vips", "%s", 
 			_( "not whitespace before start of binary data" ) );
 		return( -1 );
@@ -284,35 +331,53 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first )
 		width, height, bands, format, 
 		VIPS_CODING_NONE, interpretation, 1.0, 1.0 );
 
+	VIPS_SETSTR( out->filename, 
+		vips_stream_filename( VIPS_STREAM( streamib->streami ) ) );
+
 	return( 0 );
 }
 
+static void
+read_mmap_close_cb( VipsObject *object, VipsStreamib *streamib )
+{
+	VIPS_UNREF( streamib );
+}
+
 /* Read a ppm/pgm file using mmap().
  */
 static int
-read_mmap( FILE *fp, const char *filename, int msb_first, VipsImage *out )
+read_mmap( VipsStreamib *streamib, int msb_first, VipsImage *out )
 {
-	const guint64 header_offset = ftell( fp );
+	VipsImage **t = (VipsImage **)
-	VipsImage *x = vips_image_new();
+                vips_object_local_array( VIPS_OBJECT( out ), 3 );
-	VipsImage **t = (VipsImage **) 
+	VipsStreami *streami = streamib->streami;
-		vips_object_local_array( VIPS_OBJECT( x ), 3 );
 
-	if( vips_rawload( filename, &t[0], 
+	gint64 header_offset;
-			out->Xsize, out->Ysize, VIPS_IMAGE_SIZEOF_PEL( out ), 
+	size_t length;
-			"offset", header_offset,
+	const void *base_addr;
-			NULL ) ||
+
-		vips_copy( t[0], &t[1],
+	vips_streamib_unbuffer( streamib );
-			"bands", out->Bands, 
+	if( (header_offset = vips_streami_seek( streami, 0, SEEK_CUR )) < 0 ||
-			"format", out->BandFmt, 
+		!(base_addr = vips_streami_map( streami, &length )) )
-			"coding", out->Coding, 
-			NULL ) ||
-		vips__byteswap_bool( t[1], &t[2], 
-			vips_amiMSBfirst() != msb_first ) ||
-		vips_image_write( t[2], out ) ) {
-		g_object_unref( x );
 		return( -1 );
-	}
+
-	g_object_unref( x );
+	if( !(t[0] = vips_image_new_from_memory( 
+			base_addr + header_offset, length - header_offset,
+			out->Xsize, out->Ysize, out->Bands, out->BandFmt )) ||
+		vips_copy( t[0], &t[1],
+			"interpretation", out->Type, 
+			NULL ) ||
+		vips__byteswap_bool( t[1], &t[2],
+                        vips_amiMSBfirst() != msb_first ) ||
+		vips_image_write( t[2], out ) ) 
+		return( -1 );
+
+	/* out is using memory from the streami mmap. We need to keep a ref
+	 * until out is closed.
+	 */
+	g_object_ref( streamib );
+	g_signal_connect( out, "close", 
+		G_CALLBACK( read_mmap_close_cb ), streamib ); 
 
 	return( 0 );
 }
@@ -320,7 +385,7 @@ read_mmap( FILE *fp, const char *filename, int msb_first, VipsImage *out )
 /* Read an ascii ppm/pgm file.
  */
 static int
-read_ascii( FILE *fp, VipsImage *out )
+read_ascii( VipsStreamib *streamib, VipsImage *out )
 {
 	int x, y;
 	VipsPel *buf;
@@ -332,7 +397,7 @@ read_ascii( FILE *fp, VipsImage *out )
 		for( x = 0; x < out->Xsize * out->Bands; x++ ) {
 			int val;
 
-			if( read_int( fp, &val ) )
+			if( read_int( streamib, &val ) )
 				return( -1 );
 			
 			switch( out->BandFmt ) {
@@ -364,7 +429,7 @@ read_ascii( FILE *fp, VipsImage *out )
 /* Read an ascii 1 bit file.
  */
 static int
-read_1bit_ascii( FILE *fp, VipsImage *out )
+read_1bit_ascii( VipsStreamib *streamib, VipsImage *out )
 {
 	int x, y;
 	VipsPel *buf;
@@ -376,7 +441,7 @@ read_1bit_ascii( FILE *fp, VipsImage *out )
 		for( x = 0; x < out->Xsize * out->Bands; x++ ) {
 			int val;
 
-			if( read_int( fp, &val ) )
+			if( read_int( streamib, &val ) )
 				return( -1 );
 
 			if( val )
@@ -395,7 +460,7 @@ read_1bit_ascii( FILE *fp, VipsImage *out )
 /* Read a 1 bit binary file.
  */
 static int
-read_1bit_binary( FILE *fp, VipsImage *out )
+read_1bit_binary( VipsStreamib *streamib, VipsImage *out )
 {
 	int x, y;
 	int bits;
@@ -404,19 +469,19 @@ read_1bit_binary( FILE *fp, VipsImage *out )
 	if( !(buf = VIPS_ARRAY( out, VIPS_IMAGE_SIZEOF_LINE( out ), VipsPel )) )
 		return( -1 );
 
-	bits = fgetc( fp );
+	bits = VIPS_STREAMIB_GETC( streamib );
 	for( y = 0; y < out->Ysize; y++ ) {
 		for( x = 0; x < out->Xsize * out->Bands; x++ ) {
 			buf[x] = (bits & 128) ? 0 : 255;
 			bits = VIPS_LSHIFT_INT( bits, 1 );
 			if( (x & 7) == 7 )
-				bits = fgetc( fp );
+				bits = VIPS_STREAMIB_GETC( streamib );
 		}
 
 		/* Skip any unaligned bits at end of line.
 		 */
 		if( (x & 7) != 0 )
-			bits = fgetc( fp );
+			bits = VIPS_STREAMIB_GETC( streamib );
 
 		if( vips_image_write_line( out, y, buf ) )
 			return( -1 );
@@ -425,45 +490,39 @@ read_1bit_binary( FILE *fp, VipsImage *out )
 	return( 0 );
 }
 
-static int 
+int
-parse_ppm( FILE *fp, const char *filename, VipsImage *out )
+vips__ppm_header_stream( VipsStreamib *streamib, VipsImage *out )
 {
 	int bits;
 	int ascii;
 	int msb_first;
 
-	if( read_header( fp, out, &bits, &ascii, &msb_first ) )
+	if( read_header( streamib, out, &bits, &ascii, &msb_first ) ) 
+		return( -1 );
+
+	return( 0 );
+}
+
+int
+vips__ppm_load_stream( VipsStreamib *streamib, VipsImage *out )
+{
+	int bits;
+	int ascii;
+	int msb_first;
+
+	if( read_header( streamib, out, &bits, &ascii, &msb_first ) )
 		return( -1 );
 
 	/* What sort of read are we doing?
 	 */
 	if( !ascii && bits >= 8 )
-		return( read_mmap( fp, filename, msb_first, out ) );
+		return( read_mmap( streamib, msb_first, out ) );
 	else if( !ascii && bits == 1 )
-		return( read_1bit_binary( fp, out ) );
+		return( read_1bit_binary( streamib, out ) );
 	else if( ascii && bits == 1 )
-		return( read_1bit_ascii( fp, out ) );
+		return( read_1bit_ascii( streamib, out ) );
 	else 
-		return( read_ascii( fp, out ) );
+		return( read_ascii( streamib, out ) );
-}
-
-int
-vips__ppm_header( const char *filename, VipsImage *out )
-{
-        FILE *fp;
-	int bits;
-	int ascii;
-	int msb_first;
-
-	if( !(fp = vips__file_open_read( filename, NULL, FALSE )) ) 
-                return( -1 );
-
-	if( read_header( fp, out, &bits, &ascii, &msb_first ) ) {
-		fclose( fp );
-		return( -1 );
-	}
-
-	fclose( fp );
 
 	return( 0 );
 }
@@ -471,82 +530,54 @@ vips__ppm_header( const char *filename, VipsImage *out )
 /* Can this PPM file be read with a mmap?
  */
 static int
-isppmmmap( const char *filename )
+isppmmmap( VipsStreamib *streamib )
 {
 	VipsImage *im;
-        FILE *fp;
 	int bits;
 	int ascii;
 	int msb_first;
 
-	if( !(fp = vips__file_open_read( filename, NULL, FALSE )) ) 
-                return( -1 );
-
 	im = vips_image_new(); 
-	if( read_header( fp, im, &bits, &ascii, &msb_first ) ) {
+	if( read_header( streamib, im, &bits, &ascii, &msb_first ) ) {
 		g_object_unref( im );
-		fclose( fp );
 
 		return( 0 );
 	}
 	g_object_unref( im );
-	fclose( fp );
 
 	return( !ascii && bits >= 8 );
 }
 
-int
-vips__ppm_load( const char *filename, VipsImage *out )
-{
-        FILE *fp;
-
-	/* Note that we open in binary mode. If this is a binary PPM, we need
-	 * to be able to mmap it.
-	 */
-	if( !(fp = vips__file_open_read( filename, NULL, FALSE )) ) 
-                return( -1 );
-
-	if( parse_ppm( fp, filename, out ) ) {
-		fclose( fp );
-		return( -1 );
-	}
-
-	fclose( fp );
-
-	return( 0 );
-}
-
-int
-vips__ppm_isppm( const char *filename )
-{
-	VipsPel buf[3];
-
-	if( vips__get_bytes( filename, buf, 2 ) == 2 ) {
-		int i;
-
-		buf[2] = '\0';
-		for( i = 0; i < VIPS_NUMBER( magic_names ); i++ )
-			if( strcmp( (char *) buf, magic_names[i] ) == 0 )
-				return( TRUE );
-	}
-
-	return( 0 );
-}
-
 /* ppm flags function.
  */
 VipsForeignFlags
-vips__ppm_flags( const char *filename )
+vips__ppm_flags_stream( VipsStreamib *streamib )
 {
 	VipsForeignFlags flags;
 
 	flags = 0;
-	if( isppmmmap( filename ) )
+	if( isppmmmap( streamib ) )
 		flags |= VIPS_FOREIGN_PARTIAL;
 
 	return( flags );
 }
 
+int
+vips__ppm_isppm_stream( VipsStreami *streami )
+{
+	const unsigned char *data;
+
+	if( (data = vips_streami_sniff( streami, 2 )) ) { 
+		int i;
+
+		for( i = 0; i < VIPS_NUMBER( magic_names ); i++ )
+			if( vips_isprefix( magic_names[i], (char *) data ) )
+				return( TRUE );
+	}
+
+	return( 0 );
+}
+
 const char *vips__ppm_suffs[] = { ".ppm", ".pgm", ".pbm", ".pfm", NULL };
 
 struct _Write;

libvips/foreign/ppmload.c

@@ -66,10 +66,37 @@ typedef VipsForeignLoadClass VipsForeignLoadPpmClass;
 G_DEFINE_TYPE( VipsForeignLoadPpm, vips_foreign_load_ppm, 
 	VIPS_TYPE_FOREIGN_LOAD );
 
+static gboolean
+vips_foreign_load_ppm_is_a( const char *filename )
+{
+	VipsStreami *streami;
+	gboolean result;
+
+	if( !(streami = vips_streami_new_from_filename( filename )) )
+		return( FALSE );
+	result = vips__ppm_isppm_stream( streami );
+	VIPS_UNREF( streami );
+
+	return( result );
+}
+
 static VipsForeignFlags
 vips_foreign_load_ppm_get_flags_filename( const char *filename )
 {
-	return( (VipsForeignFlags) vips__ppm_flags( filename ) );
+	VipsStreami *streami;
+	VipsStreamib *streamib;
+	VipsForeignFlags flags;
+
+	if( !(streami = vips_streami_new_from_filename( filename )) )
+		return( 0 );
+	streamib = vips_streamib_new( streami );
+	VIPS_UNREF( streami );
+
+	flags = vips__ppm_flags_stream( streamib );
+
+	VIPS_UNREF( streamib );
+
+	return( flags );
 }
 
 static VipsForeignFlags
@@ -85,10 +112,20 @@ vips_foreign_load_ppm_header( VipsForeignLoad *load )
 {
 	VipsForeignLoadPpm *ppm = (VipsForeignLoadPpm *) load;
 
-	if( vips__ppm_header( ppm->filename, load->out ) )
+	VipsStreami *streami;
-		return( -1 );
+	VipsStreamib *streamib;
 
-	VIPS_SETSTR( load->out->filename, ppm->filename );
+	if( !(streami = vips_streami_new_from_filename( ppm->filename )) )
+		return( -1 );
+	streamib = vips_streamib_new( streami );
+	VIPS_UNREF( streami );
+
+	if( vips__ppm_header_stream( streamib, load->out ) ) {
+		VIPS_UNREF( streamib );
+		return( -1 );
+	}
+
+	VIPS_UNREF( streamib );
 
 	return( 0 );
 }
@@ -98,8 +135,20 @@ vips_foreign_load_ppm_load( VipsForeignLoad *load )
 {
 	VipsForeignLoadPpm *ppm = (VipsForeignLoadPpm *) load;
 
-	if( vips__ppm_load( ppm->filename, load->real ) )
+	VipsStreami *streami;
+	VipsStreamib *streamib;
+
+	if( !(streami = vips_streami_new_from_filename( ppm->filename )) )
 		return( -1 );
+	streamib = vips_streamib_new( streami );
+	VIPS_UNREF( streami );
+
+	if( vips__ppm_load_stream( streamib, load->real ) ) {
+		VIPS_UNREF( streamib );
+		return( -1 );
+	}
+
+	VIPS_UNREF( streamib );
 
 	return( 0 );
 }
@@ -124,7 +173,7 @@ vips_foreign_load_ppm_class_init( VipsForeignLoadPpmClass *class )
 	 */
 	foreign_class->priority = 200;
 
-	load_class->is_a = vips__ppm_isppm;
+	load_class->is_a = vips_foreign_load_ppm_is_a;
 	load_class->get_flags_filename = 
 		vips_foreign_load_ppm_get_flags_filename;
 	load_class->get_flags = vips_foreign_load_ppm_get_flags;

libvips/include/vips/stream.h

@@ -283,6 +283,10 @@ int vips_streamib_getc( VipsStreamib *streamib );
 		vips_streamib_getc( S ) \
 )
 void vips_streamib_ungetc( VipsStreamib *streamib );
+#define VIPS_STREAMIB_UNGETC( S ) { \
+	if( (S)->read_point > 0 ) \
+		(S)->read_point -= 1; \
+}
 
 int vips_streamib_require( VipsStreamib *streamib, int require );
 #define VIPS_STREAMIB_REQUIRE( S, R ) ( \

libvips/iofuncs/streamib.c

@@ -131,7 +131,7 @@ vips_streamib_unbuffer( VipsStreamib *streamib )
 	/* We'd read ahead a little way -- seek backwards by that amount.
 	 */
 	vips_streami_seek( streamib->streami, 
-		streamib->read_point - streamib->chars_in_buffer, SEEK_SET );
+		streamib->read_point - streamib->chars_in_buffer, SEEK_CUR );
 	streamib->read_point = 0;
 	streamib->chars_in_buffer = 0;
 }
@@ -171,7 +171,7 @@ vips_streamib_refill( VipsStreamib *streamib )
  *
  * Fetch the next character from the stream. 
  *
- * Use the macro VIPS_STREAMIB_GETC() instead for speed.
+ * If you can, use the macro VIPS_STREAMIB_GETC() instead for speed.
  *
  * Returns: the next char from @streamib, -1 on read error or EOF.
  */
@@ -204,6 +204,8 @@ vips_streamib_getc( VipsStreamib *streamib )
  *
  * unget more than one character is undefined. Unget at the start of the file 
  * does nothing.
+ *
+ * If you can, use the macro VIPS_STREAMIB_UNGETC() instead for speed.
  */
 void
 vips_streamib_ungetc( VipsStreamib *streamib )
@@ -212,6 +214,16 @@ vips_streamib_ungetc( VipsStreamib *streamib )
 		streamib->read_point -= 1;
 }
 
+/**
+ * VIPS_STREAMIB_UNGETC:
+ * @streamib: stream to operate on
+ *
+ * The opposite of vips_streamib_getc(): undo the previous getc.
+ *
+ * unget more than one character is undefined. Unget at the start of the file 
+ * does nothing.
+ */
+
 /**
  * vips_streamib_require:
  * @streamib: stream to operate on