From c1a027c8d747e29f134857f4cd40d1716879745c Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Thu, 14 Nov 2019 12:57:39 +0000 Subject: [PATCH] ppm load uses streams --- ChangeLog | 4 +- libvips/foreign/pforeign.h | 8 +- libvips/foreign/ppm.c | 281 +++++++++++++++++++--------------- libvips/foreign/ppmload.c | 61 +++++++- libvips/include/vips/stream.h | 4 + libvips/iofuncs/streamib.c | 16 +- 6 files changed, 235 insertions(+), 139 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9c876550..510f18ad 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,8 +17,8 @@ - nifti load/save uses double for all floating point metadata - add vips_error_buffer_copy() - add VipsStream: a universal IO class for loaders and savers -- jpeg, png, tiff (though not tiffsave), rad, svg and webp use the new IO - class +- jpeg, png, tiff (though not tiffsave), rad, svg, ppm and webp use the + new IO class - add @no_strip option to dzsave [kalozka1] - add iiif layout to dzsave - fix use of resolution-unit metadata on tiff save [kayarre] diff --git a/libvips/foreign/pforeign.h b/libvips/foreign/pforeign.h index 6409d588..90693685 100644 --- a/libvips/foreign/pforeign.h +++ b/libvips/foreign/pforeign.h @@ -143,10 +143,10 @@ int vips__mat_load( const char *filename, VipsImage *out ); int vips__mat_header( const char *filename, VipsImage *out ); int vips__mat_ismat( const char *filename ); -int vips__ppm_header( const char *name, VipsImage *out ); -int vips__ppm_load( const char *name, VipsImage *out ); -int vips__ppm_isppm( const char *filename ); -VipsForeignFlags vips__ppm_flags( const char *filename ); +int vips__ppm_header_stream( VipsStreamib *streamib, VipsImage *out ); +int vips__ppm_load_stream( VipsStreamib *streamib, VipsImage *out ); +VipsForeignFlags vips__ppm_flags_stream( VipsStreamib *streamib ); +int vips__ppm_isppm_stream( VipsStreami *streami ); extern const char *vips__ppm_suffs[]; int vips__ppm_save_stream( VipsImage *in, VipsStreamo *streamo, diff --git a/libvips/foreign/ppm.c b/libvips/foreign/ppm.c index df0289b9..582ac122 100644 --- a/libvips/foreign/ppm.c +++ b/libvips/foreign/ppm.c @@ -37,7 +37,7 @@ * 29/7/19 Kyle-Kyle * - fix a loop with malformed ppm * 13/11/19 - * - ppm save redone with streams + * - redone with streams */ /* @@ -90,36 +90,78 @@ */ #define MAX_THING (80) +/* After this, the next getc will be the first char of the next line (or EOF). + */ static void -skip_line( FILE *fp ) +skip_line( VipsStreamib *streamib ) { int ch; - while( (ch = vips__fgetc( fp )) != '\n' && + while( (ch = VIPS_STREAMIB_GETC( streamib )) != '\n' && ch != EOF ) ; } +/* After this, the next getc will be the first char of the next block of + * non-whitespace (or EOF). + */ static void -skip_white_space( FILE *fp ) +skip_white_space( VipsStreamib *streamib ) { int ch; - while( isspace( ch = vips__fgetc( fp ) ) ) + while( isspace( ch = VIPS_STREAMIB_GETC( streamib ) ) ) ; - ungetc( ch, fp ); + VIPS_STREAMIB_UNGETC( streamib ); if( ch == '#' ) { - skip_line( fp ); - skip_white_space( fp ); + skip_line( streamib ); + skip_white_space( streamib ); } } -static int -read_int( FILE *fp, int *i ) +/* After this, the next getc will be the first char of the next block of + * whitespace (or EOF). buf will be filled with up to length bytes, and + * null-terminated. + * + * If the first getc is whitespace, stop instantly and return nothing. + */ +static void +read_non_white_space( VipsStreamib *streamib, char *buf, int length ) { - skip_white_space( fp ); - if( fscanf( fp, "%d", i ) != 1 ) { + int ch; + int i; + + for( i = 0; i < length - 1 && + !isspace( ch = VIPS_STREAMIB_GETC( streamib ) ) && + ch != EOF; i++ ) + buf[i] = ch; + buf[i] = '\0'; + + /* If we stopped before seeing any whitespace, skip to the end of the + * block of non-whitespace. + */ + if( !isspace( ch ) ) + while( !isspace( ch = VIPS_STREAMIB_GETC( streamib ) ) && + ch != EOF ) + ; + + /* If we finally stopped on whitespace, step back one so the next get + * will be whitespace (or EOF). + */ + if( isspace( ch ) ) + VIPS_STREAMIB_UNGETC( streamib ); +} + +static int +read_int( VipsStreamib *streamib, int *i ) +{ + char buf[MAX_THING]; + + skip_white_space( streamib ); + read_non_white_space( streamib, buf, MAX_THING ); + + if( sscanf( buf, "%d", i ) != 1 ) { vips_error( "ppm2vips", "%s", _( "bad int" ) ); return( -1 ); } @@ -128,10 +170,14 @@ read_int( FILE *fp, int *i ) } static int -read_float( FILE *fp, float *f ) +read_float( VipsStreamib *streamib, float *f ) { - skip_white_space( fp ); - if( fscanf( fp, "%f", f ) != 1 ) { + char buf[MAX_THING]; + + skip_white_space( streamib ); + read_non_white_space( streamib, buf, MAX_THING ); + + if( sscanf( buf, "%f", f ) != 1 ) { vips_error( "ppm2vips", "%s", _( "bad float" ) ); return( -1 ); } @@ -153,7 +199,8 @@ static char *magic_names[] = { }; static int -read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) +read_header( VipsStreamib *streamib, VipsImage *out, + int *bits, int *ascii, int *msb_first ) { int width, height, bands; VipsBandFormat format; @@ -175,8 +222,8 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) /* Read in the magic number. */ - buf[0] = vips__fgetc( fp ); - buf[1] = vips__fgetc( fp ); + buf[0] = VIPS_STREAMIB_GETC( streamib ); + buf[1] = VIPS_STREAMIB_GETC( streamib ); buf[2] = '\0'; for( index = 0; index < VIPS_NUMBER( magic_names ); index++ ) @@ -196,8 +243,8 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) /* Read in size. */ - if( read_int( fp, &width ) || - read_int( fp, &height ) ) + if( read_int( streamib, &width ) || + read_int( streamib, &height ) ) return( -1 ); /* Read in max value / scale for >1 bit images. @@ -206,7 +253,7 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) if( index == 6 || index == 7 ) { float scale; - if( read_float( fp, &scale ) ) + if( read_float( streamib, &scale ) ) return( -1 ); /* Scale > 0 means big-endian. @@ -218,7 +265,7 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) else { int max_value; - if( read_int( fp, &max_value ) ) + if( read_int( streamib, &max_value ) ) return( -1 ); if( max_value > 255 ) @@ -232,7 +279,7 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) * character before the data starts. */ if( !*ascii && - !isspace( vips__fgetc( fp ) ) ) { + !isspace( VIPS_STREAMIB_GETC( streamib ) ) ) { vips_error( "ppm2vips", "%s", _( "not whitespace before start of binary data" ) ); return( -1 ); @@ -284,35 +331,53 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) width, height, bands, format, VIPS_CODING_NONE, interpretation, 1.0, 1.0 ); + VIPS_SETSTR( out->filename, + vips_stream_filename( VIPS_STREAM( streamib->streami ) ) ); + return( 0 ); } +static void +read_mmap_close_cb( VipsObject *object, VipsStreamib *streamib ) +{ + VIPS_UNREF( streamib ); +} + /* Read a ppm/pgm file using mmap(). */ static int -read_mmap( FILE *fp, const char *filename, int msb_first, VipsImage *out ) +read_mmap( VipsStreamib *streamib, int msb_first, VipsImage *out ) { - const guint64 header_offset = ftell( fp ); - VipsImage *x = vips_image_new(); - VipsImage **t = (VipsImage **) - vips_object_local_array( VIPS_OBJECT( x ), 3 ); + VipsImage **t = (VipsImage **) + vips_object_local_array( VIPS_OBJECT( out ), 3 ); + VipsStreami *streami = streamib->streami; - if( vips_rawload( filename, &t[0], - out->Xsize, out->Ysize, VIPS_IMAGE_SIZEOF_PEL( out ), - "offset", header_offset, - NULL ) || - vips_copy( t[0], &t[1], - "bands", out->Bands, - "format", out->BandFmt, - "coding", out->Coding, - NULL ) || - vips__byteswap_bool( t[1], &t[2], - vips_amiMSBfirst() != msb_first ) || - vips_image_write( t[2], out ) ) { - g_object_unref( x ); + gint64 header_offset; + size_t length; + const void *base_addr; + + vips_streamib_unbuffer( streamib ); + if( (header_offset = vips_streami_seek( streami, 0, SEEK_CUR )) < 0 || + !(base_addr = vips_streami_map( streami, &length )) ) return( -1 ); - } - g_object_unref( x ); + + if( !(t[0] = vips_image_new_from_memory( + base_addr + header_offset, length - header_offset, + out->Xsize, out->Ysize, out->Bands, out->BandFmt )) || + vips_copy( t[0], &t[1], + "interpretation", out->Type, + NULL ) || + vips__byteswap_bool( t[1], &t[2], + vips_amiMSBfirst() != msb_first ) || + vips_image_write( t[2], out ) ) + return( -1 ); + + /* out is using memory from the streami mmap. We need to keep a ref + * until out is closed. + */ + g_object_ref( streamib ); + g_signal_connect( out, "close", + G_CALLBACK( read_mmap_close_cb ), streamib ); return( 0 ); } @@ -320,7 +385,7 @@ read_mmap( FILE *fp, const char *filename, int msb_first, VipsImage *out ) /* Read an ascii ppm/pgm file. */ static int -read_ascii( FILE *fp, VipsImage *out ) +read_ascii( VipsStreamib *streamib, VipsImage *out ) { int x, y; VipsPel *buf; @@ -332,7 +397,7 @@ read_ascii( FILE *fp, VipsImage *out ) for( x = 0; x < out->Xsize * out->Bands; x++ ) { int val; - if( read_int( fp, &val ) ) + if( read_int( streamib, &val ) ) return( -1 ); switch( out->BandFmt ) { @@ -364,7 +429,7 @@ read_ascii( FILE *fp, VipsImage *out ) /* Read an ascii 1 bit file. */ static int -read_1bit_ascii( FILE *fp, VipsImage *out ) +read_1bit_ascii( VipsStreamib *streamib, VipsImage *out ) { int x, y; VipsPel *buf; @@ -376,7 +441,7 @@ read_1bit_ascii( FILE *fp, VipsImage *out ) for( x = 0; x < out->Xsize * out->Bands; x++ ) { int val; - if( read_int( fp, &val ) ) + if( read_int( streamib, &val ) ) return( -1 ); if( val ) @@ -395,7 +460,7 @@ read_1bit_ascii( FILE *fp, VipsImage *out ) /* Read a 1 bit binary file. */ static int -read_1bit_binary( FILE *fp, VipsImage *out ) +read_1bit_binary( VipsStreamib *streamib, VipsImage *out ) { int x, y; int bits; @@ -404,19 +469,19 @@ read_1bit_binary( FILE *fp, VipsImage *out ) if( !(buf = VIPS_ARRAY( out, VIPS_IMAGE_SIZEOF_LINE( out ), VipsPel )) ) return( -1 ); - bits = fgetc( fp ); + bits = VIPS_STREAMIB_GETC( streamib ); for( y = 0; y < out->Ysize; y++ ) { for( x = 0; x < out->Xsize * out->Bands; x++ ) { buf[x] = (bits & 128) ? 0 : 255; bits = VIPS_LSHIFT_INT( bits, 1 ); if( (x & 7) == 7 ) - bits = fgetc( fp ); + bits = VIPS_STREAMIB_GETC( streamib ); } /* Skip any unaligned bits at end of line. */ if( (x & 7) != 0 ) - bits = fgetc( fp ); + bits = VIPS_STREAMIB_GETC( streamib ); if( vips_image_write_line( out, y, buf ) ) return( -1 ); @@ -425,45 +490,39 @@ read_1bit_binary( FILE *fp, VipsImage *out ) return( 0 ); } -static int -parse_ppm( FILE *fp, const char *filename, VipsImage *out ) +int +vips__ppm_header_stream( VipsStreamib *streamib, VipsImage *out ) { int bits; int ascii; int msb_first; - if( read_header( fp, out, &bits, &ascii, &msb_first ) ) + if( read_header( streamib, out, &bits, &ascii, &msb_first ) ) + return( -1 ); + + return( 0 ); +} + +int +vips__ppm_load_stream( VipsStreamib *streamib, VipsImage *out ) +{ + int bits; + int ascii; + int msb_first; + + if( read_header( streamib, out, &bits, &ascii, &msb_first ) ) return( -1 ); /* What sort of read are we doing? */ if( !ascii && bits >= 8 ) - return( read_mmap( fp, filename, msb_first, out ) ); + return( read_mmap( streamib, msb_first, out ) ); else if( !ascii && bits == 1 ) - return( read_1bit_binary( fp, out ) ); + return( read_1bit_binary( streamib, out ) ); else if( ascii && bits == 1 ) - return( read_1bit_ascii( fp, out ) ); + return( read_1bit_ascii( streamib, out ) ); else - return( read_ascii( fp, out ) ); -} - -int -vips__ppm_header( const char *filename, VipsImage *out ) -{ - FILE *fp; - int bits; - int ascii; - int msb_first; - - if( !(fp = vips__file_open_read( filename, NULL, FALSE )) ) - return( -1 ); - - if( read_header( fp, out, &bits, &ascii, &msb_first ) ) { - fclose( fp ); - return( -1 ); - } - - fclose( fp ); + return( read_ascii( streamib, out ) ); return( 0 ); } @@ -471,82 +530,54 @@ vips__ppm_header( const char *filename, VipsImage *out ) /* Can this PPM file be read with a mmap? */ static int -isppmmmap( const char *filename ) +isppmmmap( VipsStreamib *streamib ) { VipsImage *im; - FILE *fp; int bits; int ascii; int msb_first; - if( !(fp = vips__file_open_read( filename, NULL, FALSE )) ) - return( -1 ); - im = vips_image_new(); - if( read_header( fp, im, &bits, &ascii, &msb_first ) ) { + if( read_header( streamib, im, &bits, &ascii, &msb_first ) ) { g_object_unref( im ); - fclose( fp ); return( 0 ); } g_object_unref( im ); - fclose( fp ); return( !ascii && bits >= 8 ); } -int -vips__ppm_load( const char *filename, VipsImage *out ) -{ - FILE *fp; - - /* Note that we open in binary mode. If this is a binary PPM, we need - * to be able to mmap it. - */ - if( !(fp = vips__file_open_read( filename, NULL, FALSE )) ) - return( -1 ); - - if( parse_ppm( fp, filename, out ) ) { - fclose( fp ); - return( -1 ); - } - - fclose( fp ); - - return( 0 ); -} - -int -vips__ppm_isppm( const char *filename ) -{ - VipsPel buf[3]; - - if( vips__get_bytes( filename, buf, 2 ) == 2 ) { - int i; - - buf[2] = '\0'; - for( i = 0; i < VIPS_NUMBER( magic_names ); i++ ) - if( strcmp( (char *) buf, magic_names[i] ) == 0 ) - return( TRUE ); - } - - return( 0 ); -} - /* ppm flags function. */ VipsForeignFlags -vips__ppm_flags( const char *filename ) +vips__ppm_flags_stream( VipsStreamib *streamib ) { VipsForeignFlags flags; flags = 0; - if( isppmmmap( filename ) ) + if( isppmmmap( streamib ) ) flags |= VIPS_FOREIGN_PARTIAL; return( flags ); } +int +vips__ppm_isppm_stream( VipsStreami *streami ) +{ + const unsigned char *data; + + if( (data = vips_streami_sniff( streami, 2 )) ) { + int i; + + for( i = 0; i < VIPS_NUMBER( magic_names ); i++ ) + if( vips_isprefix( magic_names[i], (char *) data ) ) + return( TRUE ); + } + + return( 0 ); +} + const char *vips__ppm_suffs[] = { ".ppm", ".pgm", ".pbm", ".pfm", NULL }; struct _Write; diff --git a/libvips/foreign/ppmload.c b/libvips/foreign/ppmload.c index da92b0e7..862ee780 100644 --- a/libvips/foreign/ppmload.c +++ b/libvips/foreign/ppmload.c @@ -66,10 +66,37 @@ typedef VipsForeignLoadClass VipsForeignLoadPpmClass; G_DEFINE_TYPE( VipsForeignLoadPpm, vips_foreign_load_ppm, VIPS_TYPE_FOREIGN_LOAD ); +static gboolean +vips_foreign_load_ppm_is_a( const char *filename ) +{ + VipsStreami *streami; + gboolean result; + + if( !(streami = vips_streami_new_from_filename( filename )) ) + return( FALSE ); + result = vips__ppm_isppm_stream( streami ); + VIPS_UNREF( streami ); + + return( result ); +} + static VipsForeignFlags vips_foreign_load_ppm_get_flags_filename( const char *filename ) { - return( (VipsForeignFlags) vips__ppm_flags( filename ) ); + VipsStreami *streami; + VipsStreamib *streamib; + VipsForeignFlags flags; + + if( !(streami = vips_streami_new_from_filename( filename )) ) + return( 0 ); + streamib = vips_streamib_new( streami ); + VIPS_UNREF( streami ); + + flags = vips__ppm_flags_stream( streamib ); + + VIPS_UNREF( streamib ); + + return( flags ); } static VipsForeignFlags @@ -85,10 +112,20 @@ vips_foreign_load_ppm_header( VipsForeignLoad *load ) { VipsForeignLoadPpm *ppm = (VipsForeignLoadPpm *) load; - if( vips__ppm_header( ppm->filename, load->out ) ) - return( -1 ); + VipsStreami *streami; + VipsStreamib *streamib; - VIPS_SETSTR( load->out->filename, ppm->filename ); + if( !(streami = vips_streami_new_from_filename( ppm->filename )) ) + return( -1 ); + streamib = vips_streamib_new( streami ); + VIPS_UNREF( streami ); + + if( vips__ppm_header_stream( streamib, load->out ) ) { + VIPS_UNREF( streamib ); + return( -1 ); + } + + VIPS_UNREF( streamib ); return( 0 ); } @@ -98,8 +135,20 @@ vips_foreign_load_ppm_load( VipsForeignLoad *load ) { VipsForeignLoadPpm *ppm = (VipsForeignLoadPpm *) load; - if( vips__ppm_load( ppm->filename, load->real ) ) + VipsStreami *streami; + VipsStreamib *streamib; + + if( !(streami = vips_streami_new_from_filename( ppm->filename )) ) return( -1 ); + streamib = vips_streamib_new( streami ); + VIPS_UNREF( streami ); + + if( vips__ppm_load_stream( streamib, load->real ) ) { + VIPS_UNREF( streamib ); + return( -1 ); + } + + VIPS_UNREF( streamib ); return( 0 ); } @@ -124,7 +173,7 @@ vips_foreign_load_ppm_class_init( VipsForeignLoadPpmClass *class ) */ foreign_class->priority = 200; - load_class->is_a = vips__ppm_isppm; + load_class->is_a = vips_foreign_load_ppm_is_a; load_class->get_flags_filename = vips_foreign_load_ppm_get_flags_filename; load_class->get_flags = vips_foreign_load_ppm_get_flags; diff --git a/libvips/include/vips/stream.h b/libvips/include/vips/stream.h index 4e0e0fc1..34ed9b5b 100644 --- a/libvips/include/vips/stream.h +++ b/libvips/include/vips/stream.h @@ -283,6 +283,10 @@ int vips_streamib_getc( VipsStreamib *streamib ); vips_streamib_getc( S ) \ ) void vips_streamib_ungetc( VipsStreamib *streamib ); +#define VIPS_STREAMIB_UNGETC( S ) { \ + if( (S)->read_point > 0 ) \ + (S)->read_point -= 1; \ +} int vips_streamib_require( VipsStreamib *streamib, int require ); #define VIPS_STREAMIB_REQUIRE( S, R ) ( \ diff --git a/libvips/iofuncs/streamib.c b/libvips/iofuncs/streamib.c index 5fc9fc85..bc4730a4 100644 --- a/libvips/iofuncs/streamib.c +++ b/libvips/iofuncs/streamib.c @@ -131,7 +131,7 @@ vips_streamib_unbuffer( VipsStreamib *streamib ) /* We'd read ahead a little way -- seek backwards by that amount. */ vips_streami_seek( streamib->streami, - streamib->read_point - streamib->chars_in_buffer, SEEK_SET ); + streamib->read_point - streamib->chars_in_buffer, SEEK_CUR ); streamib->read_point = 0; streamib->chars_in_buffer = 0; } @@ -171,7 +171,7 @@ vips_streamib_refill( VipsStreamib *streamib ) * * Fetch the next character from the stream. * - * Use the macro VIPS_STREAMIB_GETC() instead for speed. + * If you can, use the macro VIPS_STREAMIB_GETC() instead for speed. * * Returns: the next char from @streamib, -1 on read error or EOF. */ @@ -204,6 +204,8 @@ vips_streamib_getc( VipsStreamib *streamib ) * * unget more than one character is undefined. Unget at the start of the file * does nothing. + * + * If you can, use the macro VIPS_STREAMIB_UNGETC() instead for speed. */ void vips_streamib_ungetc( VipsStreamib *streamib ) @@ -212,6 +214,16 @@ vips_streamib_ungetc( VipsStreamib *streamib ) streamib->read_point -= 1; } +/** + * VIPS_STREAMIB_UNGETC: + * @streamib: stream to operate on + * + * The opposite of vips_streamib_getc(): undo the previous getc. + * + * unget more than one character is undefined. Unget at the start of the file + * does nothing. + */ + /** * vips_streamib_require: * @streamib: stream to operate on