revise decode/unminimise use

This commit is contained in:
John Cupitt 2019-10-28 05:04:04 +00:00
parent cbbd4ae6dd
commit ba0100eb13
9 changed files with 208 additions and 184 deletions

View File

@ -1030,20 +1030,23 @@ char *
build_scan_properties( VipsImage *image )
{
VipsDbuf dbuf;
GTimeVal now;
GDateTime *now;
char *date;
int i;
now = g_date_time_new_now_local();
date = g_date_time_format_iso8601( now );
g_date_time_unref( now );
vips_dbuf_init( &dbuf );
g_get_current_time( &now );
date = g_time_val_to_iso8601( &now );
vips_dbuf_writef( &dbuf, "<?xml version=\"1.0\"?>\n" );
vips_dbuf_writef( &dbuf, "<image xmlns=\"http://www.pathozoom.com/szi\""
" date=\"%s\" version=\"1.0\">\n", date );
g_free( date );
vips_dbuf_writef( &dbuf, " <properties>\n" );
g_free( date );
for( i = 0; i < VIPS_NUMBER( scan_property_names ); i++ )
build_scan_property( &dbuf, image,
scan_property_names[i][0],

View File

@ -558,7 +558,8 @@ read_jpeg_header( ReadJpeg *jpeg, VipsImage *out )
interpretation,
xres, yres );
VIPS_SETSTR( out->filename, VIPS_STREAM( jpeg->input )->filename );
VIPS_SETSTR( out->filename,
vips_stream_filename( VIPS_STREAM( jpeg->input ) ) );
vips_image_pipelinev( out, VIPS_DEMAND_STYLE_FATSTRIP, NULL );
@ -743,11 +744,6 @@ read_jpeg_generate( VipsRegion *or,
g_thread_self(), r->top, r->height );
#endif /*DEBUG_VERBOSE*/
/* In pixel decode mode.
*/
if( vips_streami_decode( jpeg->input ) )
return( -1 );
VIPS_GATE_START( "read_jpeg_generate: work" );
/* We're inside a tilecache where tiles are the full image width, so
@ -953,8 +949,6 @@ vips__jpeg_read( ReadJpeg *jpeg, VipsImage *out, gboolean header_only )
*/
vips_autorot_remove_angle( out );
}
vips_streami_minimise( jpeg->input );
}
else {
if( read_jpeg_image( jpeg, out ) )
@ -976,10 +970,9 @@ vips__jpeg_read_stream( VipsStreami *input, VipsImage *out,
if( setjmp( jpeg->eman.jmp ) )
return( -1 );
if( readjpeg_open_input( jpeg ) )
return( -1 );
if( vips__jpeg_read( jpeg, out, header_only ) )
if( readjpeg_open_input( jpeg ) ||
vips__jpeg_read( jpeg, out, header_only ) ||
vips_streami_decode( jpeg->input ) )
return( -1 );
return( 0 );

View File

@ -534,7 +534,8 @@ rtiff_new( VipsStreami *input, VipsImage *out,
g_signal_connect( out, "minimise",
G_CALLBACK( rtiff_minimise_cb ), rtiff );
if( rtiff->page < 0 || rtiff->page > 1000000 ) {
if( rtiff->page < 0 ||
rtiff->page > 1000000 ) {
vips_error( "tiff2vips", _( "bad page number %d" ),
rtiff->page );
return( NULL );
@ -1367,6 +1368,9 @@ rtiff_set_header( Rtiff *rtiff, VipsImage *out )
out->Xsize = rtiff->header.width;
out->Ysize = rtiff->header.height * rtiff->n;
VIPS_SETSTR( out->filename,
vips_stream_filename( VIPS_STREAM( rtiff->input ) ) );
if( rtiff->n > 1 )
vips_image_set_int( out,
VIPS_META_PAGE_HEIGHT, rtiff->header.height );
@ -1534,9 +1538,6 @@ rtiff_fill_region( VipsRegion *out,
int x, y, z;
if( vips_streami_unminimise( rtiff->input ) )
return( -1 );
/* Special case: we are filling a single tile exactly sized to match
* the tiff tile and we have no repacking to do for this format.
*/
@ -1863,9 +1864,6 @@ rtiff_stripwise_generate( VipsRegion *or,
g_assert( r->height ==
VIPS_MIN( read_height, or->im->Ysize - r->top ) );
if( vips_streami_unminimise( rtiff->input ) )
return( -1 );
/* And check that y_pos is correct. It should be, since we are inside
* a vips_sequential().
*/

View File

@ -198,8 +198,6 @@ vips_foreign_load_tiff_file_header( VipsForeignLoad *load )
}
VIPS_UNREF( input );
VIPS_SETSTR( load->out->filename, file->filename );
return( 0 );
}

View File

@ -481,7 +481,8 @@ png2vips_header( Read *read, VipsImage *out )
VIPS_CODING_NONE, interpretation,
Xres, Yres );
VIPS_SETSTR( out->filename, VIPS_STREAM( read->input )->filename );
VIPS_SETSTR( out->filename,
vips_stream_filename( VIPS_STREAM( read->input ) ) );
/* Uninterlaced images will be read in seq mode. Interlaced images are
* read via a huge memory buffer.
@ -621,11 +622,6 @@ png2vips_generate( VipsRegion *or,
return( -1 );
}
/* In pixel decode mode.
*/
if( vips_streami_decode( read->input ) )
return( -1 );
for( y = 0; y < r->height; y++ ) {
png_bytep q = (png_bytep) VIPS_REGION_ADDR( or, 0, r->top + y );
@ -715,11 +711,10 @@ vips__png_header_stream( VipsStreami *input, VipsImage *out )
Read *read;
if( !(read = read_new( input, out, TRUE )) ||
png2vips_header( read, out ) )
png2vips_header( read, out ) ||
vips_streami_decode( input ) )
return( -1 );
vips_streami_minimise( input );
return( 0 );
}
@ -729,8 +724,9 @@ vips__png_read_stream( VipsStreami *input, VipsImage *out, gboolean fail )
Read *read;
if( !(read = read_new( input, out, fail )) ||
png2vips_image( read, out ) )
return( -1 );
png2vips_image( read, out ) ||
vips_streami_decode( input ) )
return( -1 );
return( 0 );
}

View File

@ -88,7 +88,8 @@ GType vips_stream_get_type( void );
int vips_stream_open( VipsStream *stream );
int vips_stream_close( VipsStream *stream );
const char *vips_stream_name( VipsStream *stream );
const char *vips_stream_filename( VipsStream *stream );
const char *vips_stream_nick( VipsStream *stream );
#define VIPS_TYPE_STREAMI (vips_streami_get_type())
#define VIPS_STREAMI( obj ) \
@ -204,13 +205,13 @@ VipsStreami *vips_streami_new_from_blob( VipsBlob *blob );
VipsStreami *vips_streami_new_from_memory( const void *data, size_t size );
VipsStreami *vips_streami_new_from_options( const char *options );
void vips_streami_minimise( VipsStreami *streami );
int vips_streami_unminimise( VipsStreami *streami );
int vips_streami_decode( VipsStreami *streami );
ssize_t vips_streami_read( VipsStreami *streami, void *data, size_t length );
const void *vips_streami_map( VipsStreami *streami, size_t *length );
gint64 vips_streami_seek( VipsStreami *streami, gint64 offset, int whence );
int vips_streami_rewind( VipsStreami *streami );
void vips_streami_minimise( VipsStreami *streami );
int vips_streami_unminimise( VipsStreami *streami );
int vips_streami_decode( VipsStreami *streami );
unsigned char *vips_streami_sniff( VipsStreami *streami, size_t length );
gint64 vips_streami_size( VipsStreami *streami );

View File

@ -143,10 +143,15 @@ vips_stream_init( VipsStream *stream )
}
const char *
vips_stream_name( VipsStream *stream )
vips_stream_filename( VipsStream *stream )
{
return( stream->filename );
}
const char *
vips_stream_nick( VipsStream *stream )
{
return( stream->filename ?
stream->filename :
VIPS_OBJECT( stream )->nickname );
}

View File

@ -34,15 +34,11 @@
/* TODO
*
* - filename encoding
* - move fds in here
* - add vips_stream_nick() to make stream nickname
* - something to stop unbounded streams filling memory
* - gaussblur is missing the vector path again argh
* - can we map and then close the fd? how about on Windows?
* - are we detecting EOF correctly? what about interrupted reads? perhaps
* we should check errno as well
* - need to be able to set is_pipe via constructor
* - need open()/close() as vfuncs?
* - make a subclass that lets you set vfuncs as params
* - make a subclass that lets you set vfuncs as params, inc. close(),
* is_pipe etc.
*/
/*
@ -94,6 +90,7 @@
G_DEFINE_TYPE( VipsStreami, vips_streami, VIPS_TYPE_STREAM );
#ifdef VIPS_DEBUG
static void
vips_streami_sanity( VipsStreami *streami )
{
@ -182,6 +179,13 @@ vips_streami_sanity( VipsStreami *streami )
VIPS_STREAM( streami )->descriptor) );
}
}
#endif /*VIPS_DEBUG*/
#ifdef VIPS_DEBUG
#define SANITY( S ) vips_streami_sanity( S )
#else /*!VIPS_DEBUG*/
#define SANITY( S )
#endif /*VIPS_DEBUG*/
static void
vips_streami_finalize( GObject *gobject )
@ -213,7 +217,7 @@ vips_streami_build( VipsObject *object )
if( vips_object_argument_isset( object, "filename" ) &&
vips_object_argument_isset( object, "descriptor" ) ) {
vips_error( vips_stream_name( stream ),
vips_error( vips_stream_nick( stream ),
"%s", _( "don't set 'filename' and 'descriptor'" ) );
return( -1 );
}
@ -269,9 +273,16 @@ vips_streami_read_real( VipsStreami *streami, void *data, size_t length )
{
VipsStream *stream = VIPS_STREAM( streami );
ssize_t bytes_read;
VIPS_DEBUG_MSG( "vips_streami_read_real:\n" );
return( read( stream->descriptor, data, length ) );
do {
bytes_read = read( stream->descriptor, data, length );
} while( bytes_read < 0 && errno == EINTR );
return( bytes_read );
}
static gint64
@ -354,7 +365,7 @@ vips_streami_new_from_descriptor( int descriptor )
return( NULL );
}
vips_streami_sanity( streami );
SANITY( streami );
return( streami );
}
@ -384,7 +395,7 @@ vips_streami_new_from_filename( const char *filename )
return( NULL );
}
vips_streami_sanity( streami );
SANITY( streami );
return( streami );
}
@ -413,7 +424,7 @@ vips_streami_new_from_blob( VipsBlob *blob )
return( NULL );
}
vips_streami_sanity( streami );
SANITY( streami );
return( streami );
}
@ -446,7 +457,7 @@ vips_streami_new_from_memory( const void *data, size_t length )
vips_area_unref( VIPS_AREA( blob ) );
vips_streami_sanity( streami );
SANITY( streami );
return( streami );
}
@ -474,11 +485,123 @@ vips_streami_new_from_options( const char *options )
return( NULL );
}
vips_streami_sanity( streami );
SANITY( streami );
return( streami );
}
/**
* vips_streami_minimise:
* @streami: input stream to operate on
*
* Minimise the stream. As many stream resources as can be safely removed are
* removed. Use vips_streami_unminimise() to restore the stream if you wish to
* use it again.
*
* Loaders should call this in response to the minimise signal on their output
* image.
*
* Returns: 0 on success, or -1 on error.
*/
void
vips_streami_minimise( VipsStreami *streami )
{
VipsStream *stream = VIPS_STREAM( streami );
VIPS_DEBUG_MSG( "vips_streami_minimise:\n" );
SANITY( streami );
if( stream->filename &&
stream->descriptor != -1 &&
stream->tracked_descriptor == stream->descriptor &&
!streami->is_pipe ) {
VIPS_DEBUG_MSG( " tracked_close()\n" );
vips_tracked_close( stream->tracked_descriptor );
stream->tracked_descriptor = -1;
stream->descriptor = -1;
}
SANITY( streami );
}
/**
* vips_streami_unminimise:
* @streami: input stream to operate on
*
* Restore the stream after minimisation. This is called at the start
* of every stream method, so loaders should not usually need this.
*
* See also: vips_streami_minimise().
*
* Returns: 0 on success, or -1 on error.
*/
int
vips_streami_unminimise( VipsStreami *streami )
{
VipsStream *stream = VIPS_STREAM( streami );
VIPS_DEBUG_MSG( "vips_streami_unminimise:\n" );
if( stream->descriptor == -1 &&
stream->tracked_descriptor == -1 &&
stream->filename ) {
int fd;
if( (fd = vips_tracked_open( stream->filename,
MODE_READ )) == -1 )
return( -1 );
stream->tracked_descriptor = fd;
stream->descriptor = fd;
VIPS_DEBUG_MSG( "vips_streami_unminimise: "
"restoring read position %zd\n",
streami->read_position );
if( vips__seek( stream->descriptor,
streami->read_position, SEEK_SET ) == -1 )
return( -1 );
}
return( 0 );
}
/**
* vips_streami_decode:
* @streami: input stream to operate on
*
* Signal the end of header read and the start of the pixel decode phase.
* After this, you can no longer seek on this stream.
*
* Loaders should call this at the end of header read.
*
* See also: vips_streami_unminimise().
*
* Returns: 0 on success, -1 on error.
*/
int
vips_streami_decode( VipsStreami *streami )
{
VIPS_DEBUG_MSG( "vips_streami_decode:\n" );
SANITY( streami );
/* We have finished reading the header. We can discard the header bytes
* we saved.
*/
if( !streami->decode ) {
streami->decode = TRUE;
VIPS_FREEF( g_byte_array_unref, streami->header_bytes );
VIPS_FREEF( g_byte_array_unref, streami->sniff );
}
vips_streami_minimise( streami );
SANITY( streami );
return( 0 );
}
/**
* vips_streami_read:
* @streami: input stream to operate on
@ -502,7 +625,10 @@ vips_streami_read( VipsStreami *streami, void *buffer, size_t length )
VIPS_DEBUG_MSG( "vips_streami_read:\n" );
vips_streami_sanity( streami );
SANITY( streami );
if( vips_streami_unminimise( streami ) )
return( -1 );
bytes_read = 0;
@ -551,7 +677,7 @@ vips_streami_read( VipsStreami *streami, void *buffer, size_t length )
VIPS_DEBUG_MSG( " %zd bytes from read()\n", n );
if( n == -1 ) {
vips_error_system( errno,
vips_stream_name(
vips_stream_nick(
VIPS_STREAM( streami ) ),
"%s", _( "read error" ) );
return( -1 );
@ -574,7 +700,7 @@ vips_streami_read( VipsStreami *streami, void *buffer, size_t length )
VIPS_DEBUG_MSG( " %zd bytes total\n", bytes_read );
vips_streami_sanity( streami );
SANITY( streami );
return( bytes_read );
}
@ -588,7 +714,8 @@ vips_streami_pipe_read_to_position( VipsStreami *streami, gint64 target )
gint64 old_read_position;
unsigned char buffer[4096];
VIPS_DEBUG_MSG( "vips_streami_pipe_read_position:\n" );
VIPS_DEBUG_MSG( "vips_streami_pipe_read_position: %" G_GINT64_FORMAT
"\n", target );
g_assert( !streami->decode );
g_assert( streami->header_bytes );
@ -596,14 +723,15 @@ vips_streami_pipe_read_to_position( VipsStreami *streami, gint64 target )
if( target < 0 ||
(streami->length != -1 &&
target > streami->length) ) {
vips_error( vips_stream_name( VIPS_STREAM( streami ) ),
vips_error( vips_stream_nick( VIPS_STREAM( streami ) ),
_( "bad read to %" G_GINT64_FORMAT ), target );
return( -1 );
}
old_read_position = streami->read_position;
/* TODO ... add something to prevent unbounded streams filling memory.
/* TODO ... add something to prevent unbounded streams filling memory
* if target == -1.
*/
while( target == -1 ||
streami->read_position < target ) {
@ -688,7 +816,10 @@ vips_streami_map( VipsStreami *streami, size_t *length_out )
VIPS_DEBUG_MSG( "vips_streami_map:\n" );
vips_streami_sanity( streami );
SANITY( streami );
if( vips_streami_unminimise( streami ) )
return( NULL );
/* Pipes need to be converted to memory streams.
*/
@ -709,7 +840,7 @@ vips_streami_map( VipsStreami *streami, size_t *length_out )
if( length_out )
*length_out = streami->length;
vips_streami_sanity( streami );
SANITY( streami );
return( streami->data );
}
@ -735,7 +866,10 @@ vips_streami_seek( VipsStreami *streami, gint64 offset, int whence )
VIPS_DEBUG_MSG( "vips_streami_seek: offset = %" G_GINT64_FORMAT
", whence = %d\n", offset, whence );
vips_streami_sanity( streami );
SANITY( streami );
if( vips_streami_unminimise( streami ) )
return( -1 );
if( streami->data ) {
switch( whence ) {
@ -752,7 +886,7 @@ vips_streami_seek( VipsStreami *streami, gint64 offset, int whence )
break;
default:
vips_error( vips_stream_name( VIPS_STREAM( streami ) ),
vips_error( vips_stream_nick( VIPS_STREAM( streami ) ),
"%s", _( "bad 'whence'" ) );
return( -1 );
break;
@ -780,7 +914,7 @@ vips_streami_seek( VipsStreami *streami, gint64 offset, int whence )
break;
default:
vips_error( vips_stream_name( VIPS_STREAM( streami ) ),
vips_error( vips_stream_nick( VIPS_STREAM( streami ) ),
"%s", _( "bad 'whence'" ) );
return( -1 );
break;
@ -799,14 +933,14 @@ vips_streami_seek( VipsStreami *streami, gint64 offset, int whence )
if( new_pos < 0 ||
(streami->length != -1 &&
new_pos > streami->length) ) {
vips_error( vips_stream_name( VIPS_STREAM( streami ) ),
vips_error( vips_stream_nick( VIPS_STREAM( streami ) ),
_( "bad seek to %" G_GINT64_FORMAT ), new_pos );
return( -1 );
}
streami->read_position = new_pos;
vips_streami_sanity( streami );
SANITY( streami );
VIPS_DEBUG_MSG( " new_pos = %" G_GINT64_FORMAT "\n", new_pos );
@ -827,82 +961,12 @@ vips_streami_rewind( VipsStreami *streami )
{
VIPS_DEBUG_MSG( "vips_streami_rewind:\n" );
vips_streami_sanity( streami );
SANITY( streami );
if( vips_streami_seek( streami, 0, SEEK_SET ) != 0 )
return( -1 );
vips_streami_sanity( streami );
return( 0 );
}
/**
* vips_streami_minimise:
* @streami: input stream to operate on
*
* Minimise the stream. As many stream resources as can be safely removed are
* removed. Use vips_streami_unminimise() to restore the stream if you wish to
* use it again.
*
* Returns: 0 on success, or -1 on error.
*/
void
vips_streami_minimise( VipsStreami *streami )
{
VipsStream *stream = VIPS_STREAM( streami );
VIPS_DEBUG_MSG( "vips_streami_minimise:\n" );
vips_streami_sanity( streami );
if( stream->filename &&
stream->descriptor != -1 &&
stream->tracked_descriptor == stream->descriptor &&
!streami->is_pipe ) {
VIPS_DEBUG_MSG( " tracked_close()\n" );
vips_tracked_close( stream->tracked_descriptor );
stream->tracked_descriptor = -1;
stream->descriptor = -1;
}
vips_streami_sanity( streami );
}
/**
* vips_streami_unminimise:
* @streami: input stream to operate on
*
* Restore the stream after minimisation.
*
* Returns: 0 on success, or -1 on error.
*/
int
vips_streami_unminimise( VipsStreami *streami )
{
VipsStream *stream = VIPS_STREAM( streami );
VIPS_DEBUG_MSG( "vips_streami_unminimise:\n" );
if( stream->descriptor == -1 &&
stream->tracked_descriptor == -1 &&
stream->filename ) {
int fd;
if( (fd = vips_tracked_open( stream->filename,
MODE_READ )) == -1 )
return( -1 );
stream->tracked_descriptor = fd;
stream->descriptor = fd;
VIPS_DEBUG_MSG( "vips_streami_unminimise: "
"restoring read position %zd\n",
streami->read_position );
if( vips__seek( stream->descriptor,
streami->read_position, SEEK_SET ) == -1 )
return( -1 );
}
SANITY( streami );
return( 0 );
}
@ -925,52 +989,17 @@ vips_streami_size( VipsStreami *streami )
VIPS_DEBUG_MSG( "vips_streami_size:\n" );
vips_streami_sanity( streami );
SANITY( streami );
read_position = vips_streami_seek( streami, 0, SEEK_CUR );
size = vips_streami_seek( streami, 0, SEEK_END );
vips_streami_seek( streami, read_position, SEEK_SET );
vips_streami_sanity( streami );
SANITY( streami );
return( size );
}
/**
* vips_streami_decode:
* @streami: input stream to operate on
*
* Signal the end of header read and the start of the pixel decode phase. After
* this, you can no longer seek on this stream.
*
* Returns: 0 on success, -1 on error.
*/
int
vips_streami_decode( VipsStreami *streami )
{
VIPS_DEBUG_MSG( "vips_streami_decode:\n" );
vips_streami_sanity( streami );
/* We have finished reading the header. We can discard the bytes we
* saved.
*/
if( !streami->decode ) {
streami->decode = TRUE;
VIPS_FREEF( g_byte_array_unref, streami->header_bytes );
VIPS_FREEF( g_byte_array_unref, streami->sniff );
}
/* Make sure we are open, in case we've been minimised.
*/
if( vips_streami_unminimise( streami ) )
return( -1 );
vips_streami_sanity( streami );
return( 0 );
}
/**
* vips_streami_sniff:
* @streami: sniff this stream
@ -986,9 +1015,10 @@ vips_streami_sniff( VipsStreami *streami, size_t length )
VIPS_DEBUG_MSG( "vips_streami_sniff: %zd bytes\n", length );
vips_streami_sanity( streami );
SANITY( streami );
if( vips_streami_rewind( streami ) )
if( vips_streami_unminimise( streami ) ||
vips_streami_rewind( streami ) )
return( NULL );
g_byte_array_set_size( streami->sniff, length );
@ -998,7 +1028,7 @@ vips_streami_sniff( VipsStreami *streami, size_t length )
n == 0 )
return( NULL );
vips_streami_sanity( streami );
SANITY( streami );
return( streami->sniff->data );
}

View File

@ -117,7 +117,7 @@ vips_streamo_build( VipsObject *object )
if( vips_object_argument_isset( object, "filename" ) &&
vips_object_argument_isset( object, "descriptor" ) ) {
vips_error( vips_stream_name( stream ),
vips_error( vips_stream_nick( stream ),
"%s", _( "don't set 'filename' and 'descriptor'" ) );
return( -1 );
}
@ -131,7 +131,7 @@ vips_streamo_build( VipsObject *object )
*/
if( (fd = vips_tracked_open( filename,
MODE_WRITE, 0644 )) == -1 ) {
vips_error_system( errno, vips_stream_name( stream ),
vips_error_system( errno, vips_stream_nick( stream ),
"%s", _( "unable to open for write" ) );
return( -1 );
}
@ -330,7 +330,7 @@ vips_streamo_write( VipsStreamo *streamo, const void *data, size_t length )
*/
if( n <= 0 ) {
vips_error_system( errno,
vips_stream_name( VIPS_STREAM( streamo ) ),
vips_stream_nick( VIPS_STREAM( streamo ) ),
"%s", _( "write error" ) );
return( -1 );
}