From fb2ab23e26043162440ae297cf369380f4625707 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Wed, 16 Oct 2019 18:28:46 +0100 Subject: [PATCH] start adding TIFF stream load streams now have a seek() vfunc --- libvips/foreign/pforeign.h | 7 +++ libvips/foreign/tiff.c | 94 +++++++++++++++++++++++++++++ libvips/foreign/tiff.h | 2 + libvips/foreign/tiff2vips.c | 100 +++++++++++++++++++++++++++++++ libvips/include/vips/stream.h | 40 +++++++++---- libvips/iofuncs/stream.c | 109 ++++++++++++++++++++++++++-------- 6 files changed, 316 insertions(+), 36 deletions(-) diff --git a/libvips/foreign/pforeign.h b/libvips/foreign/pforeign.h index 6fa33e1c..3c6be4b6 100644 --- a/libvips/foreign/pforeign.h +++ b/libvips/foreign/pforeign.h @@ -95,6 +95,13 @@ int vips__tiff_read_header_buffer( const void *buf, size_t len, VipsImage *out, int vips__tiff_read_buffer( const void *buf, size_t len, VipsImage *out, int page, int n, gboolean autorotate ); +gboolean vips__istiff_stream( VipsStreamInput *input ); +gboolean vips__istifftiled_stream( VipsStreamInput *input ); +int vips__tiff_read_header_stream( VipsStreamInput *input, VipsImage *out, + int page, int n, gboolean autorotate ); +int vips__tiff_read_stream( VipsStreamInput *input, VipsImage *out, + int page, int n, gboolean autorotate ); + extern const char *vips__foreign_tiff_suffs[]; int vips__isanalyze( const char *filename ); diff --git a/libvips/foreign/tiff.c b/libvips/foreign/tiff.c index b231a5ba..b0fe5339 100644 --- a/libvips/foreign/tiff.c +++ b/libvips/foreign/tiff.c @@ -300,6 +300,100 @@ vips__tiff_openin_buffer( VipsImage *image, const void *data, size_t length ) return( tiff ); } +/* TIFF input from a vips stream. + */ + +static tsize_t +openin_stream_read( thandle_t st, tdata_t data, tsize_t size ) +{ + VipsStreamInput *input = (VipsStreamInput *) st; + + return( vips_stream_input_read( input, data, size ) ); +} + +static tsize_t +openin_stream_write( thandle_t st, tdata_t buffer, tsize_t size ) +{ + g_assert_not_reached(); + + return( 0 ); +} + +static toff_t +openin_stream_seek( thandle_t st, toff_t position, int whence ) +{ + VipsStreamInput *input = (VipsStreamInput *) st; + + return( vips_stream_input_seek( input, position, whence ) ); +} + +static int +openin_stream_close( thandle_t st ) +{ + VipsStreamInput *input = (VipsStreamInput *) st; + + VIPS_UNREF( input ); + + return( 0 ); +} + +static toff_t +openin_stream_size( thandle_t st ) +{ + /* Do we need this? + */ + printf( "aaaaargh!!\n" ); + g_assert( FALSE ); + + return( 0 ); +} + +static int +openin_stream_map( thandle_t st, tdata_t *start, toff_t *len ) +{ + g_assert_not_reached(); + + return( 0 ); +} + +static void +openin_stream_unmap( thandle_t st, tdata_t start, toff_t len ) +{ + g_assert_not_reached(); + + return; +} + +TIFF * +vips__tiff_openin_stream( VipsImage *image, VipsStreamInput *input ) +{ + TIFF *tiff; + +#ifdef DEBUG + printf( "vips__tiff_openin_buffer:\n" ); +#endif /*DEBUG*/ + + /* Unreffed on close(), see above. + */ + g_object_ref( input ); + + if( !(tiff = TIFFClientOpen( "stream input", "rm", + (thandle_t) input, + openin_stream_read, + openin_stream_write, + openin_stream_seek, + openin_stream_close, + openin_stream_size, + openin_stream_map, + openin_stream_unmap )) ) { + vips_error( "vips__tiff_openin_stream", "%s", + _( "unable to open stream for input" ) ); + return( NULL ); + } + + return( tiff ); +} + /* TIFF output to a memory buffer. */ diff --git a/libvips/foreign/tiff.h b/libvips/foreign/tiff.h index e86f8c37..b1681dd2 100644 --- a/libvips/foreign/tiff.h +++ b/libvips/foreign/tiff.h @@ -45,6 +45,8 @@ TIFF *vips__tiff_openin_buffer( VipsImage *image, TIFF *vips__tiff_openout_buffer( VipsImage *image, gboolean bigtiff, void **out_data, size_t *out_length ); +TIFF *vips__tiff_openin_stream( VipsImage *image, VipsStreamInput *input ); + #ifdef __cplusplus } #endif /*__cplusplus*/ diff --git a/libvips/foreign/tiff2vips.c b/libvips/foreign/tiff2vips.c index 245f9807..01d0fbf6 100644 --- a/libvips/foreign/tiff2vips.c +++ b/libvips/foreign/tiff2vips.c @@ -2591,4 +2591,104 @@ vips__istifftiled_buffer( const void *buf, size_t len ) return( tiled ); } +static gboolean +vips__testtiff_stream( VipsStreamInput *input, TiffPropertyFn fn ) +{ + VipsImage *im; + TIFF *tif; + gboolean property; + + vips__tiff_init(); + + im = vips_image_new(); + + if( vips_stream_input_rewind( input ) ) + return( FALSE ); + if( !(tif = vips__tiff_openin_stream( im, input )) ) { + g_object_unref( im ); + vips_error_clear(); + return( FALSE ); + } + + property = fn ? fn( tif ) : TRUE; + + TIFFClose( tif ); + g_object_unref( im ); + + return( property ); +} + +gboolean +vips__istiff_stream( VipsStreamInput *input ) +{ + return( vips__testtiff_stream( input, NULL ) ); +} + +gboolean +vips__istifftiled_stream( VipsStreamInput *input ) +{ + return( vips__testtiff_stream( input, TIFFIsTiled ) ); +} + +static Rtiff * +rtiff_new_stream( VipsStreamInput *input, VipsImage *out, + int page, int n, gboolean autorotate ) +{ + Rtiff *rtiff; + + if( !(rtiff = rtiff_new( out, page, n, autorotate )) || + !(rtiff->tiff = vips__tiff_openin_stream( out, input )) || + rtiff_header_read_all( rtiff ) ) + return( NULL ); + + return( rtiff ); +} + +int +vips__tiff_read_header_stream( VipsStreamInput *input, VipsImage *out, + int page, int n, gboolean autorotate ) +{ + Rtiff *rtiff; + + vips__tiff_init(); + + if( !(rtiff = rtiff_new_stream( input, out, page, n, autorotate )) ) + return( -1 ); + + if( rtiff_set_header( rtiff, out ) ) + return( -1 ); + + vips__tiff_read_header_orientation( rtiff, out ); + + return( 0 ); +} + +int +vips__tiff_read_stream( VipsStreamInput *input, VipsImage *out, + int page, int n, gboolean autorotate ) +{ + Rtiff *rtiff; + +#ifdef DEBUG + printf( "tiff2vips: libtiff version is \"%s\"\n", TIFFGetVersion() ); + printf( "tiff2vips: libtiff starting for buffer %p\n", buf ); +#endif /*DEBUG*/ + + vips__tiff_init(); + + if( !(rtiff = rtiff_new_stream( input, out, page, n, autorotate )) ) + return( -1 ); + + if( rtiff->header.tiled ) { + if( rtiff_read_tilewise( rtiff, out ) ) + return( -1 ); + } + else { + if( rtiff_read_stripwise( rtiff, out ) ) + return( -1 ); + } + + return( 0 ); +} + #endif /*HAVE_TIFF*/ diff --git a/libvips/include/vips/stream.h b/libvips/include/vips/stream.h index 152e9087..f54ef887 100644 --- a/libvips/include/vips/stream.h +++ b/libvips/include/vips/stream.h @@ -106,7 +106,7 @@ const char *vips_stream_filename( VipsStream *stream ); /* Read from something like a socket, file or memory area and present the data * with a simple seek / read interface. * - * During the header phase, we save data from unseekable sources in a buffer + * During the header phase, we save data from unseekable streams in a buffer * so readers can rewind and read again. We don't buffer data during the * decode stage. */ @@ -116,7 +116,7 @@ typedef struct _VipsStreamInput { /* We have two phases: * * During the header phase, we save bytes read from the input (if this - * is an unseekable source) so that we can rewind and try again, if + * is an unseekable stream) so that we can rewind and try again, if * necessary. * * Once we reach decode phase, we no longer support rewind and the @@ -124,13 +124,13 @@ typedef struct _VipsStreamInput { */ gboolean decode; - /* TRUE is this descriptor supports lseek(). If not, then we save data - * read during header phase in a buffer. + /* TRUE if this descriptor supports lseek(). If not, then we save data + * read during the header phase in a buffer. */ gboolean seekable; - /* TRUE is this descriptor supports mmap(). If not, then we have to - * read() the whole thing. + /* TRUE if this descriptor supports mmap(). If not, then we have to + * read() the whole stream if the loader needs the entire image. */ gboolean mapable; @@ -149,11 +149,11 @@ typedef struct _VipsStreamInput { */ GByteArray *sniff; - /* For a memory source, the blob we read from. + /* For a memory stream, the blob we read from. */ VipsBlob *blob; - /* If we've mmaped the file, the base and length. + /* If we've mmaped the file, the base and length of the mapped area. */ const void *baseaddr; size_t length; @@ -165,9 +165,27 @@ typedef struct _VipsStreamInputClass { /* Subclasses can define these to implement other input methods. */ + + /* Read up to N bytes from the stream into the supplied buffer, + * returning the number of bytes actually read. + * + * -1 on error, 0 on EOF. + */ ssize_t (*read)( VipsStreamInput *, unsigned char *, size_t ); - const void * (*map)( VipsStreamInput *, size_t * ); - int (*rewind)( VipsStreamInput * ); + + /* Map the entire stream into memory, for example with mmap(). Return + * the base and size of the mapped area. + * + * If this is not defined, the file will be read in with repeated + * calls to ->read(). + * + * NULL on error. + */ + const void *(*map)( VipsStreamInput *, size_t * ); + + /* Seek to a certain position, args exactly as lseek(2). + */ + off_t (*seek)( VipsStreamInput *, off_t offset, int ); /* Shut down anything that can safely restarted. For example, if * there's a fd that supports lseek(), it can be closed, since later @@ -192,6 +210,8 @@ VipsStreamInput *vips_stream_input_new_from_options( const char *options ); ssize_t vips_stream_input_read( VipsStreamInput *input, unsigned char *data, size_t length ); const void *vips_stream_input_map( VipsStreamInput *input, size_t *length ); +off_t vips_stream_input_seek( VipsStreamInput *input, + off_t offset, int whence ); int vips_stream_input_rewind( VipsStreamInput *input ); void vips_stream_input_minimise( VipsStreamInput *input ); void vips_stream_input_decode( VipsStreamInput *input ); diff --git a/libvips/iofuncs/stream.c b/libvips/iofuncs/stream.c index a0f6060b..e3f29547 100644 --- a/libvips/iofuncs/stream.c +++ b/libvips/iofuncs/stream.c @@ -345,29 +345,29 @@ vips_stream_input_map_real( VipsStreamInput *input, size_t *length ) return( file_baseaddr ); } -static int -vips_stream_input_rewind_real( VipsStreamInput *input ) +static off_t +vips_stream_input_seek_real( VipsStreamInput *input, off_t offset, int whence ) { VipsStream *stream = VIPS_STREAM( input ); - VIPS_DEBUG_MSG( "vips_stream_input_rewind_real:\n" ); + off_t new_pos; - if( input->seekable && - stream->descriptor != -1 ) { - off_t new_pos; + VIPS_DEBUG_MSG( "vips_stream_input_seek_real:\n" ); - VIPS_DEBUG_MSG( " rewinding desriptor %d\n", - stream->descriptor ); - - new_pos = lseek( stream->descriptor, 0, SEEK_SET ); - if( new_pos == -1 ) { - vips_error_system( errno, STREAM_NAME( stream ), - "%s", _( "unable to rewind" ) ); - return( 0 ); - } + if( !input->seekable || + stream->descriptor == -1 ) { + vips_error( STREAM_NAME( stream ), "%s", _( "not seekable" ) ); + return( -1 ); } - return( 0 ); + new_pos = lseek( stream->descriptor, offset, whence ); + if( new_pos == -1 ) { + vips_error_system( errno, STREAM_NAME( stream ), + "%s", _( "seek error" ) ); + return( -1 ); + } + + return( new_pos ); } static void @@ -398,7 +398,7 @@ vips_stream_input_class_init( VipsStreamInputClass *class ) class->read = vips_stream_input_read_real; class->map = vips_stream_input_map_real; - class->rewind = vips_stream_input_rewind_real; + class->seek = vips_stream_input_seek_real; class->minimise = vips_stream_input_minimise_real; VIPS_ARG_BOXED( class, "blob", 3, @@ -671,22 +671,79 @@ vips_stream_input_map( VipsStreamInput *input, size_t *length ) return( input->header_bytes->data ); } -int -vips_stream_input_rewind( VipsStreamInput *input ) +off_t +vips_stream_input_seek( VipsStreamInput *input, off_t offset, int whence ) { VipsStreamInputClass *class = VIPS_STREAM_INPUT_GET_CLASS( input ); - VIPS_DEBUG_MSG( "vips_stream_input_rewind:\n" ); + off_t new_pos; - if( input->decode ) { - vips_error( STREAM_NAME( input ), - "%s", _( "can't rewind after decode begins" ) ); - return( -1 ); + VIPS_DEBUG_MSG( "vips_stream_input_seek:\n" ); + + switch( whence ) { + case SEEK_SET: + new_pos = offset; + break; + + case SEEK_CUR: + new_pos = input->read_position + offset; + break; + + case SEEK_END: + /* TODO .. I don't think any loader needs SEEK_END. + */ + default: + vips_error( STREAM_NAME( input ), "%s", _( "bad 'whence'" ) ); + return( -1 ); + break; } - input->read_position = 0; + if( !input->seekable ) { + /* We can seek on non-seekable streams during the header phase. + */ + if( input->decode ) { + vips_error( STREAM_NAME( input ), + "%s", _( "can't rewind after decode begins" ) ); + return( -1 ); + } - return( class->rewind( input ) ); + g_assert( input->header_bytes ); + + /* We may not have read up to the new position. + */ + while( input->read_position < new_pos ) { + unsigned char buffer[4096]; + + if( vips_stream_input_read( input, buffer, 4096 ) ) + return( -1 ); + } + } + else if( input->blob || + input->baseaddr ) { + /* Memory streams and mapped streams don't need a seek. + */ + ; + } + else { + new_pos = class->seek( input, offset, whence ); + if( new_pos == -1 ) + return( -1 ); + } + + input->read_position = new_pos; + + return( new_pos ); +} + +int +vips_stream_input_rewind( VipsStreamInput *input ) +{ + VIPS_DEBUG_MSG( "vips_stream_input_rewind:\n" ); + + if( vips_stream_input_seek( input, 0, SEEK_SET ) != 0 ) + return( -1 ); + + return( 0 ); } void