From fc5a4a917ad74f35935ce836f66cedfcccbc08e5 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Thu, 26 Feb 2015 14:09:01 +0000 Subject: [PATCH] make file format readers close fds early we were using two fds per open image, now we just use one jpg, magick, tiff, png --- ChangeLog | 1 + TODO | 1 + libvips/foreign/jpeg2vips.c | 13 ++++++++++++- libvips/foreign/magick2vips.c | 25 +++++++++++++++++++++---- libvips/foreign/tiff2vips.c | 19 +++++++++++++++++-- libvips/foreign/vipspng.c | 9 +++++++++ 6 files changed, 61 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index e4f84e6c..03cb7d1d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ - add magicload_buffer() [mcuelenaere] - added test_foreign.py, plus more test images - rewritten tiff writer is about 3 - 4x faster at making pyramids +- jpg, magick, png, tiff readers now use only 1 fd per input image 6/2/15 started 7.42.3 - bump version for back-compat ABI change diff --git a/TODO b/TODO index d3347ba9..64dcc15c 100644 --- a/TODO +++ b/TODO @@ -1,3 +1,4 @@ + - support orientation tag in tiff images see diff --git a/libvips/foreign/jpeg2vips.c b/libvips/foreign/jpeg2vips.c index a5fb2610..faaaf89a 100644 --- a/libvips/foreign/jpeg2vips.c +++ b/libvips/foreign/jpeg2vips.c @@ -64,6 +64,9 @@ * 20/1/15 * - don't call jpeg_finish_decompress(), all it does is read and check * the tail of the file + * 26/2/15 + * - close the jpeg read down early for a header read ... this saves an + * fd during jpg read, handy for large numbers of input images */ /* @@ -166,6 +169,8 @@ typedef struct _ReadJpeg { gboolean autorotate; } ReadJpeg; +/* This can be called many times. + */ static int readjpeg_free( ReadJpeg *jpeg ) { @@ -199,7 +204,7 @@ readjpeg_free( ReadJpeg *jpeg ) VIPS_FREE( jpeg->filename ); jpeg->eman.fp = NULL; - /* I don't think this can fail. + /* I don't think this can fail. It's harmless to call many times. */ jpeg_destroy_decompress( &jpeg->cinfo ); @@ -1111,6 +1116,12 @@ vips__jpeg_read_file( const char *filename, VipsImage *out, if( vips__jpeg_read( jpeg, out, header_only ) ) return( -1 ); + /* We can kill off the decompress early if this is just a header read. + * This saves an fd during read. + */ + if( header_only ) + readjpeg_free( jpeg ); + return( 0 ); } diff --git a/libvips/foreign/magick2vips.c b/libvips/foreign/magick2vips.c index 9dffc9ea..962764fb 100644 --- a/libvips/foreign/magick2vips.c +++ b/libvips/foreign/magick2vips.c @@ -44,6 +44,9 @@ * - add @density option * 16/2/15 mcuelenaere * - add blob read + * 26/2/15 + * - close the read down early for a header read ... this saves an + * fd during file read, handy for large numbers of input images */ /* @@ -127,11 +130,13 @@ typedef struct _Read { GMutex *lock; } Read; -static int -read_destroy( VipsImage *im, Read *read ) +/* Can be called many times. + */ +static void +read_free( Read *read ) { #ifdef DEBUG - printf( "magick2vips: read_destroy: %s\n", read->filename ); + printf( "magick2vips: read_free: %s\n", read->filename ); #endif /*DEBUG*/ VIPS_FREE( read->filename ); @@ -140,6 +145,14 @@ read_destroy( VipsImage *im, Read *read ) VIPS_FREE( read->frames ); DestroyExceptionInfo( &read->exception ); VIPS_FREEF( vips_g_mutex_free, read->lock ); +} + +/* Can be called many times. + */ +static int +read_close( VipsImage *im, Read *read ) +{ + read_free( read ); return( 0 ); } @@ -173,7 +186,7 @@ read_new( const char *filename, VipsImage *im, gboolean all_frames, read->frame_height = 0; read->lock = vips_g_mutex_new(); - g_signal_connect( im, "close", G_CALLBACK( read_destroy ), read ); + g_signal_connect( im, "close", G_CALLBACK( read_close ), read ); if( !read->image_info ) return( NULL ); @@ -741,6 +754,10 @@ vips__magick_read_header( const char *filename, VipsImage *im, return( -1 ); } + /* Just a header read: we can free the read early and save an fd. + */ + read_free( read ); + return( 0 ); } diff --git a/libvips/foreign/tiff2vips.c b/libvips/foreign/tiff2vips.c index a36d9dc8..ec4a3c21 100644 --- a/libvips/foreign/tiff2vips.c +++ b/libvips/foreign/tiff2vips.c @@ -154,6 +154,9 @@ * - read any XMP metadata * 19/1/15 * - try to handle 8-bit colormaps + * 26/2/15 + * - close the read down early for a header read ... this saves an + * fd during file read, handy for large numbers of input images */ /* @@ -1678,12 +1681,20 @@ read_stripwise( ReadTiff *rtiff, VipsImage *out ) return( 0 ); } +/* Can be called many times. + */ static void -readtiff_destroy( VipsObject *object, ReadTiff *rtiff ) +readtiff_free( ReadTiff *rtiff ) { VIPS_FREEF( TIFFClose, rtiff->tiff ); } +static void +readtiff_close( VipsObject *object, ReadTiff *rtiff ) +{ + readtiff_free( rtiff ); +} + static ReadTiff * readtiff_new( VipsImage *out, int page, gboolean readbehind ) { @@ -1710,7 +1721,7 @@ readtiff_new( VipsImage *out, int page, gboolean readbehind ) rtiff->contig_buf = NULL; g_signal_connect( out, "close", - G_CALLBACK( readtiff_destroy ), rtiff ); + G_CALLBACK( readtiff_close ), rtiff ); if( rtiff->page < 0 || rtiff->page > 1000000 ) { vips_error( "tiff2vips", _( "bad page number %d" ), @@ -1918,6 +1929,10 @@ vips__tiff_read_header( const char *filename, VipsImage *out, int page ) if( parse_header( rtiff, out ) ) return( -1 ); + /* Just a header read: we can free the tiff read early and save an fd. + */ + readtiff_free( rtiff ); + return( 0 ); } diff --git a/libvips/foreign/vipspng.c b/libvips/foreign/vipspng.c index 256a2431..5f854e0b 100644 --- a/libvips/foreign/vipspng.c +++ b/libvips/foreign/vipspng.c @@ -52,6 +52,9 @@ * - don't check profiles, helps with libpng >=1.6.11 * 27/10/14 Lovell * - add @filter option + * 26/2/15 + * - close the read down early for a header read ... this saves an + * fd during file read, handy for large numbers of input images */ /* @@ -150,6 +153,8 @@ typedef struct { } Read; +/* Can be called many times. + */ static void read_destroy( Read *read ) { @@ -423,6 +428,10 @@ vips__png_header( const char *name, VipsImage *out ) png2vips_header( read, out ) ) return( -1 ); + /* Just a header read: we can free the read early and save an fd. + */ + read_destroy( read ); + return( 0 ); }