From 387cafa738592070f6a4b2bca2aa1e7f16924642 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Thu, 10 Oct 2019 20:42:39 +0100 Subject: [PATCH] add jpegload_stream --- libvips/foreign/foreign.c | 30 +++++++++ libvips/foreign/jpeg2vips.c | 114 +++++++++++++++++++++++++++++++++ libvips/foreign/jpegload.c | 78 ++++++++++++++++++++++ libvips/foreign/pforeign.h | 4 ++ libvips/include/vips/foreign.h | 9 +++ libvips/include/vips/stream.h | 16 +++-- libvips/iofuncs/stream.c | 87 +++++++++++++++++++++---- 7 files changed, 322 insertions(+), 16 deletions(-) diff --git a/libvips/foreign/foreign.c b/libvips/foreign/foreign.c index a48591ad..4135e15c 100644 --- a/libvips/foreign/foreign.c +++ b/libvips/foreign/foreign.c @@ -457,6 +457,8 @@ vips_foreign_load_summary_class( VipsObjectClass *object_class, VipsBuf *buf ) vips_buf_appends( buf, ", is_a" ); if( class->is_a_buffer ) vips_buf_appends( buf, ", is_a_buffer" ); + if( class->is_a_stream ) + vips_buf_appends( buf, ", is_a_stream" ); if( class->get_flags ) vips_buf_appends( buf, ", get_flags" ); if( class->get_flags_filename ) @@ -672,6 +674,32 @@ vips_foreign_is_a_buffer( const char *loader, const void *data, size_t size ) return( FALSE ); } +/** + * vips_foreign_is_a_stream: + * @loader: name of loader to use for test + * @input: stream to test + * + * Return %TRUE if @input can be loaded by @loader. @loader is something + * like "tiffload_stream" or "VipsForeignLoadTiffStream". + * + * Returns: %TRUE if @data can be loaded by @stream. + */ +gboolean +vips_foreign_is_a_stream( const char *loader, VipsStreamInput *input ) +{ + const VipsObjectClass *class; + VipsForeignLoadClass *load_class; + + if( !(class = vips_class_find( "VipsForeignLoad", loader )) ) + return( FALSE ); + load_class = VIPS_FOREIGN_LOAD_CLASS( class ); + if( load_class->is_a_stream && + load_class->is_a_stream( input ) ) + return( TRUE ); + + return( FALSE ); +} + /** * vips_foreign_flags: * @loader: name of loader to use for test @@ -1869,6 +1897,7 @@ vips_foreign_operation_init( void ) extern GType vips_foreign_load_openslide_get_type( void ); extern GType vips_foreign_load_jpeg_file_get_type( void ); extern GType vips_foreign_load_jpeg_buffer_get_type( void ); + extern GType vips_foreign_load_jpeg_stream_get_type( void ); extern GType vips_foreign_save_jpeg_file_get_type( void ); extern GType vips_foreign_save_jpeg_buffer_get_type( void ); extern GType vips_foreign_save_jpeg_mime_get_type( void ); @@ -1980,6 +2009,7 @@ vips_foreign_operation_init( void ) #ifdef HAVE_JPEG vips_foreign_load_jpeg_file_get_type(); vips_foreign_load_jpeg_buffer_get_type(); + vips_foreign_load_jpeg_stream_get_type(); vips_foreign_save_jpeg_file_get_type(); vips_foreign_save_jpeg_buffer_get_type(); vips_foreign_save_jpeg_mime_get_type(); diff --git a/libvips/foreign/jpeg2vips.c b/libvips/foreign/jpeg2vips.c index 3ac7c6ee..150ae7b5 100644 --- a/libvips/foreign/jpeg2vips.c +++ b/libvips/foreign/jpeg2vips.c @@ -202,6 +202,10 @@ typedef struct _ReadJpeg { const void *buf; size_t len; + /* The stream we read from. + */ + VipsStreamInput *input; + } ReadJpeg; /* Private struct for memory input. @@ -296,6 +300,61 @@ term_source( j_decompress_ptr cinfo ) { } +#define STREAM_BUFFER_SIZE (4096) + +/* Private struct for stream input. + */ +typedef struct { + /* Public jpeg fields. + */ + struct jpeg_source_mgr pub; + + /* Private stuff during read. + */ + VipsStreamInput *input; + unsigned char buf[STREAM_BUFFER_SIZE]; + +} InputSource; + +static void +stream_init_source( j_decompress_ptr cinfo ) +{ + InputSource *source = (InputSource *) cinfo->src; + + /* Start off empty ... libjpeg will call fill_input_buffer to get the + * first bytes. + */ + source->pub.next_input_byte = source->buf; + source->pub.bytes_in_buffer = 0; +} + +/* Fill the input buffer --- called whenever buffer is emptied. + */ +static boolean +stream_fill_input_buffer( j_decompress_ptr cinfo ) +{ + static const JOCTET eoi_buffer[4] = { + (JOCTET) 0xFF, (JOCTET) JPEG_EOI, 0, 0 + }; + + InputSource *source = (InputSource *) cinfo->src; + + size_t read; + + if( (read = vips_stream_input_read( source->input, + source->buf, STREAM_BUFFER_SIZE )) == -1 ) { + WARNMS( cinfo, JWRN_JPEG_EOF ); + source->pub.next_input_byte = eoi_buffer; + source->pub.bytes_in_buffer = 2; + } + else { + source->pub.next_input_byte = source->buf; + source->pub.bytes_in_buffer = read; + } + + return( TRUE ); +} + static int readjpeg_open_input( ReadJpeg *jpeg ) { @@ -333,6 +392,27 @@ readjpeg_open_input( ReadJpeg *jpeg ) src->pub.next_input_byte = jpeg->buf; } + if( jpeg->input && + !cinfo->src ) { + InputSource *src; + + cinfo->src = (struct jpeg_source_mgr *) + (*cinfo->mem->alloc_small)( + (j_common_ptr) cinfo, JPOOL_PERMANENT, + sizeof( InputSource ) ); + + src = (InputSource *) cinfo->src; + src->input = jpeg->input; + + src->pub.init_source = stream_init_source; + src->pub.fill_input_buffer = stream_fill_input_buffer; + src->pub.skip_input_data = skip_input_data; + src->pub.resync_to_restart = jpeg_resync_to_restart; + src->pub.term_source = term_source; + src->pub.bytes_in_buffer = 0; + src->pub.next_input_byte = src->buf; + } + return( 0 ); } @@ -1117,4 +1197,38 @@ vips__isjpeg( const char *filename ) return( 0 ); } +int +vips__jpeg_read_stream( VipsStreamInput *input, VipsImage *out, + gboolean header_only, int shrink, int fail, gboolean autorotate ) +{ + ReadJpeg *jpeg; + + if( !(jpeg = readjpeg_new( out, shrink, fail, autorotate )) ) + return( -1 ); + + if( setjmp( jpeg->eman.jmp ) ) + return( -1 ); + + jpeg->input = input; + if( readjpeg_open_input( jpeg ) ) + return( -1 ); + + if( vips__jpeg_read( jpeg, out, header_only ) ) + return( -1 ); + + return( 0 ); +} + +int +vips__isjpeg_stream( VipsStreamInput *input ) +{ + const unsigned char *p; + + if( (p = vips_stream_input_sniff( input, 2 )) && + vips__isjpeg_buffer( p, 2 ) ) + return( 1 ); + + return( 0 ); +} + #endif /*HAVE_JPEG*/ diff --git a/libvips/foreign/jpegload.c b/libvips/foreign/jpegload.c index 742ee435..ad74f3c0 100644 --- a/libvips/foreign/jpegload.c +++ b/libvips/foreign/jpegload.c @@ -315,6 +315,84 @@ vips_foreign_load_jpeg_buffer_init( VipsForeignLoadJpegBuffer *buffer ) { } +typedef struct _VipsForeignLoadJpegStream { + VipsForeignLoadJpeg parent_object; + + /* Load from a buffer. + */ + VipsStreamInput *input; + +} VipsForeignLoadJpegStream; + +typedef VipsForeignLoadJpegClass VipsForeignLoadJpegStreamClass; + +G_DEFINE_TYPE( VipsForeignLoadJpegStream, vips_foreign_load_jpeg_stream, + vips_foreign_load_jpeg_get_type() ); + +static int +vips_foreign_load_jpeg_stream_header( VipsForeignLoad *load ) +{ + VipsForeignLoadJpeg *jpeg = (VipsForeignLoadJpeg *) load; + VipsForeignLoadJpegStream *stream = (VipsForeignLoadJpegStream *) load; + + if( vips__jpeg_read_stream( stream->input, + load->out, TRUE, jpeg->shrink, load->fail, jpeg->autorotate ) ) + return( -1 ); + + return( 0 ); +} + +static int +vips_foreign_load_jpeg_stream_load( VipsForeignLoad *load ) +{ + VipsForeignLoadJpeg *jpeg = (VipsForeignLoadJpeg *) load; + VipsForeignLoadJpegStream *stream = (VipsForeignLoadJpegStream *) load; + + if( vips__jpeg_read_stream( stream->input, + load->real, FALSE, jpeg->shrink, load->fail, + jpeg->autorotate ) ) + return( -1 ); + + return( 0 ); +} + +static gboolean +vips_foreign_load_jpeg_stream_is_a( VipsStreamInput *input ) +{ + return( vips__isjpeg_stream( input ) ); +} + +static void +vips_foreign_load_jpeg_stream_class_init( + VipsForeignLoadJpegStreamClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *object_class = (VipsObjectClass *) class; + VipsForeignLoadClass *load_class = (VipsForeignLoadClass *) class; + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + object_class->nickname = "jpegload_stream"; + object_class->description = _( "load jpeg from stream" ); + + load_class->is_a_stream = vips_foreign_load_jpeg_stream_is_a; + load_class->header = vips_foreign_load_jpeg_stream_header; + load_class->load = vips_foreign_load_jpeg_stream_load; + + VIPS_ARG_BOXED( class, "input", 1, + _( "Input" ), + _( "Input stream to load from" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadJpegStream, input ), + VIPS_TYPE_STREAM_INPUT ); +} + +static void +vips_foreign_load_jpeg_stream_init( VipsForeignLoadJpegStream *stream ) +{ +} + #endif /*HAVE_JPEG*/ /** diff --git a/libvips/foreign/pforeign.h b/libvips/foreign/pforeign.h index 40f79e26..234fa0bd 100644 --- a/libvips/foreign/pforeign.h +++ b/libvips/foreign/pforeign.h @@ -192,6 +192,10 @@ int vips__jpeg_read_file( const char *name, VipsImage *out, int vips__jpeg_read_buffer( const void *buf, size_t len, VipsImage *out, gboolean header_only, int shrink, int fail, gboolean autorotate ); +int vips__jpeg_read_stream( VipsStreamInput *input, VipsImage *out, + gboolean header_only, int shrink, int fail, gboolean autorotate ); +int vips__isjpeg_stream( VipsStreamInput *input ); + int vips__png_header( const char *name, VipsImage *out ); int vips__png_read( const char *name, VipsImage *out, gboolean fail ); gboolean vips__png_ispng_buffer( const void *buf, size_t len ); diff --git a/libvips/include/vips/foreign.h b/libvips/include/vips/foreign.h index 432394fc..b7819b60 100644 --- a/libvips/include/vips/foreign.h +++ b/libvips/include/vips/foreign.h @@ -184,6 +184,13 @@ typedef struct _VipsForeignLoadClass { */ gboolean (*is_a_buffer)( const void *data, size_t size ); + /* Is a stream in this format. + * + * This function should return %TRUE if the stream contains an image of + * this type. + */ + gboolean (*is_a_stream)( VipsStreamInput *stream ); + /* Get the flags from a filename. * * This function should examine the file and return a set @@ -238,6 +245,8 @@ VipsForeignFlags vips_foreign_flags( const char *loader, const char *filename ); gboolean vips_foreign_is_a( const char *loader, const char *filename ); gboolean vips_foreign_is_a_buffer( const char *loader, const void *data, size_t size ); +gboolean vips_foreign_is_a_stream( const char *loader, + VipsStreamInput *stream ); void vips_foreign_load_invalidate( VipsImage *image ); diff --git a/libvips/include/vips/stream.h b/libvips/include/vips/stream.h index 221a2ccb..c2b17bc3 100644 --- a/libvips/include/vips/stream.h +++ b/libvips/include/vips/stream.h @@ -140,6 +140,10 @@ typedef struct _VipsStreamInput { */ GByteArray *header_bytes; + /* Save the first few bytes here for file type sniffing. + */ + GByteArray *sniff; + /* For a memory source, the blob we read from. */ VipsBlob *blob; @@ -169,7 +173,7 @@ VipsStreamInput *vips_stream_input_new_from_memory( void *data, size_t size ); ssize_t vips_stream_input_read( VipsStreamInput *input, unsigned char *buffer, size_t length ); -void vips_stream_input_rewind( VipsStreamInput *input ); +int vips_stream_input_rewind( VipsStreamInput *input ); void vips_stream_input_decode( VipsStreamInput *input ); gboolean vips_stream_input_eof( VipsStreamInput *input ); unsigned char *vips_stream_input_sniff( VipsStreamInput *input, size_t length ); @@ -195,7 +199,11 @@ typedef struct _VipsStreamOutput { VipsStream parent_object; /*< private >*/ - + + /* For memory output, the blob we write to. + */ + VipsBlob *blob; + } VipsStreamOutput; typedef struct _VipsStreamOutputClass { @@ -209,8 +217,8 @@ typedef struct _VipsStreamOutputClass { GType vips_stream_output_get_type( void ); -VipsStreamOutput *vips_stream_output_new_from_descriptor( int descriptor ); -VipsStreamOutput *vips_stream_output_new_from_filename( const char *filename ); +VipsStreamOutput *vips_stream_output_new_to_descriptor( int descriptor ); +VipsStreamOutput *vips_stream_output_new_to_filename( const char *filename ); int vips_stream_output_write( VipsStreamOutput *stream, const unsigned char *buffer, size_t buffer_size ); diff --git a/libvips/iofuncs/stream.c b/libvips/iofuncs/stream.c index 62a43803..32192651 100644 --- a/libvips/iofuncs/stream.c +++ b/libvips/iofuncs/stream.c @@ -31,6 +31,15 @@ */ +/* TODO + * + * - memory output + * - add mmapable descriptors + * - add seekable descriptors + * - can we really change all behaviour in the subclass? will we need map and + * seek as well as read and rewind? + */ + /* #define VIPS_DEBUG */ @@ -171,6 +180,7 @@ vips_stream_input_finalize( GObject *gobject ) VipsStreamInput *input = VIPS_STREAM_INPUT( gobject ); VIPS_FREEF( g_byte_array_unref, input->header_bytes ); + VIPS_FREEF( g_byte_array_unref, input->sniff ); G_OBJECT_CLASS( vips_stream_input_parent_class )->finalize( gobject ); } @@ -215,11 +225,15 @@ vips_stream_input_build( VipsObject *object ) if( vips_object_argument_isset( object, "blob" ) ) input->rewindable = TRUE; - /* We will need a save buffer for unrewindable streams. + /* Need to save the header if the source is not rewindable. */ - if( !input->rewindable ) + if( !input->rewindable ) input->header_bytes = g_byte_array_new(); + /* We always want a sniff buffer. + */ + input->sniff = g_byte_array_new(); + return( 0 ); } @@ -483,10 +497,11 @@ vips_stream_input_read( VipsStreamInput *input, if( read == 0 ) input->eof = TRUE; - /* If we are in header mode and there's a save buffer, we need - * to store this new data in case we rewind. + /* If we're not rewindable, we need to save header bytes for + * reuse. */ if( input->header_bytes && + !input->rewindable && !input->decode && read > 0 ) g_byte_array_append( input->header_bytes, @@ -499,6 +514,14 @@ vips_stream_input_read( VipsStreamInput *input, return( bytes_read ); } +int +vips_stream_input_rewind( VipsStreamInput *input ) +{ + VipsStreamInputClass *class = VIPS_STREAM_INPUT_GET_CLASS( input ); + + return( class->rewind( input ) ); +} + gboolean vips_stream_input_eof( VipsStreamInput *input ) { @@ -510,6 +533,7 @@ vips_stream_input_decode( VipsStreamInput *input ) { input->decode = TRUE; VIPS_FREEF( g_byte_array_unref, input->header_bytes ); + VIPS_FREEF( g_byte_array_unref, input->sniff ); } /** @@ -519,9 +543,24 @@ vips_stream_input_decode( VipsStreamInput *input ) * Return a pointer to the first few bytes of the file. */ unsigned char * -vips_stream_input_sniff( VipsStreamInput *stream, size_t length ) +vips_stream_input_sniff( VipsStreamInput *input, size_t length ) { - return( NULL ); + ssize_t read; + unsigned char *q; + + if( vips_stream_input_rewind( input ) ) + return( NULL ); + + g_byte_array_set_size( input->sniff, length ); + + for( q = input->sniff->data; length > 0; length -= read, q += read ) { + read = vips_stream_input_read( input, q, length ); + if( read == -1 || + read == 0 ) + return( NULL ); + } + + return( input->sniff->data ); } G_DEFINE_TYPE( VipsStreamOutput, vips_stream_output, VIPS_TYPE_STREAM ); @@ -529,38 +568,62 @@ G_DEFINE_TYPE( VipsStreamOutput, vips_stream_output, VIPS_TYPE_STREAM ); static int vips_stream_output_build( VipsObject *object ) { - VipsStreamOutput *stream = VIPS_STREAM_OUTPUT( object ); + VipsStream *stream = VIPS_STREAM( object ); + VipsStreamOutput *output = VIPS_STREAM_OUTPUT( object ); - VIPS_DEBUG_MSG( "vips_stream_output_build: %p\n", stream ); + VIPS_DEBUG_MSG( "vips_stream_output_build: %p\n", output ); if( VIPS_OBJECT_CLASS( vips_stream_output_parent_class )-> build( object ) ) return( -1 ); if( vips_object_argument_isset( object, "filename" ) && - !vips_object_argument_isset( object, "descriptor" ) ) { - const char *filename = VIPS_STREAM( stream )->filename; + vips_object_argument_isset( object, "descriptor" ) ) { + vips_error( STREAM_NAME( stream ), + "%s", _( "don't set 'filename' and 'descriptor'" ) ); + return( -1 ); + } + + if( vips_object_argument_isset( object, "filename" ) ) { + const char *filename = stream->filename; int fd; if( (fd = vips_tracked_open( filename, MODE_WRITE )) == -1 ) { - vips_error_system( errno, filename, + vips_error_system( errno, STREAM_NAME( stream ), "%s", _( "unable to open for write" ) ); return( -1 ); } - g_object_set( object, "descriptor", fd, NULL ); + stream->tracked_descriptor = fd; + stream->descriptor = fd; } + if( vips_object_argument_isset( object, "descriptor" ) ) + stream->close_descriptor = stream->descriptor; + return( 0 ); } static void vips_stream_output_class_init( VipsStreamOutputClass *class ) { + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); VipsObjectClass *vobject_class = VIPS_OBJECT_CLASS( class ); + gobject_class->finalize = vips_stream_input_finalize; + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + vobject_class->build = vips_stream_output_build; + + VIPS_ARG_BOXED( class, "blob", 1, + _( "Blob" ), + _( "Blob to save to" ), + VIPS_ARGUMENT_OPTIONAL_OUTPUT, + G_STRUCT_OFFSET( VipsStreamOutput, blob ), + VIPS_TYPE_BLOB ); + } static void