generic sequential read mode

moved sequential read stuff into VipsForeignLoad
This commit is contained in:
John Cupitt 2012-02-15 13:55:34 +00:00
parent 71d55c4554
commit aee2999259
9 changed files with 296 additions and 143 deletions

View File

@ -3,7 +3,7 @@
- added vips_foreign_find_save_options()/vips_foreign_find_load_options()
- delayed write to foreign via a "w" image was not working
- support operations with many returns in Python
- sequential read mode for pngload
- sequential read mode
- better im_shrink()
20/8/11 started 7.27.0

3
TODO
View File

@ -1,7 +1,6 @@
- add a sequential mode to all readers
we can't do this neatly in the base load class until we move all loaders to
classes ... do an ad-hoc hack for now
tiff, jpg are the obvious ones
- argh

View File

@ -1,4 +1,7 @@
/* VIPS function dispatch tables for image file load/save.
*
* 7/2/12
* - add support for sequential reads
*/
/*
@ -28,8 +31,8 @@
*/
/*
#define DEBUG
*/
#define DEBUG
#ifdef HAVE_CONFIG_H
#include <config.h>
@ -62,7 +65,7 @@
* process, loading an image from a file or saving an image to a file. These
* functions let you give load or save options as name - value pairs in the C
* argument list. You can use vips_foreign_load_options() and
* vips_foreign_save_options() to include option in the file name.
* vips_foreign_save_options() to include options in the file name.
*
* For example:
*
@ -75,6 +78,12 @@
* Will save an image to the file "frank.tiff" in TIFF format (selected by
* the file name extension) with JPEG compression.
*
* |[
* vips_foreign_save_options (my_image, "frank.tiff[compression=jpeg]");
* ]|
*
* Is the same thing, but with the option in the filename.
*
* You can also invoke the operations directly, for example:
*
* |[
@ -100,6 +109,7 @@
* VipsForeignFlags:
* @VIPS_FOREIGN_NONE: no flags set
* @VIPS_FOREIGN_PARTIAL: the image may be read lazilly
* @VIPS_FOREIGN_SEQUENTIAL: top-to-bottom lazy reading
* @VIPS_FOREIGN_BIGENDIAN: image pixels are most-significant byte first
*
* Some hints about the image loader.
@ -107,6 +117,13 @@
* @VIPS_FOREIGN_PARTIAL means that the image can be read directly from the
* file without needing to be unpacked to a temporary image first.
*
* @VIPS_FOREIGN_SEQUENTIAL means that the loader supports lazy reading, but
* only top-to-bottom (sequential) access. Formats like PNG can read sets of
* scanlines, for example, but only in order.
*
* If neither PARTIAL or SEQUENTIAL is set, the loader only supports whole
* image read. Setting both PARTIAL and SEQUENTIAL is an error.
*
* @VIPS_FOREIGN_BIGENDIAN means that image pixels are most-significant byte
* first. Depending on the native byte order of the host machine, you may
* need to swap bytes. See copy_swap().
@ -124,7 +141,8 @@
/**
* VipsForeignLoad:
*
* @out must be set by @header(). @load(), if defined, must set @real.
* @header() must set at least the header fields of @out. @laod(), if defined,
* must load the pixels to @real.
*/
/**
@ -151,7 +169,15 @@ G_DEFINE_TYPE( VipsForeignLoadPng, vips_foreign_load_png,
static VipsForeignFlags
vips_foreign_load_png_get_flags_filename( const char *filename )
{
return( 0 );
VipsForeignFlags flags;
flags = 0;
if( vips__png_isinterlaced( filename ) )
flags = VIPS_FOREIGN_PARTIAL;
else
flags = VIPS_FOREIGN_SEQUENTIAL;
return( flags );
}
static VipsForeignFlags
@ -644,13 +670,32 @@ vips_get_disc_threshold( void )
if( vips__disc_threshold )
threshold = vips__parse_size( vips__disc_threshold );
VIPS_DEBUG_MSG( "vips_get_disc_threshold: "
"%zd bytes\n", threshold );
#ifdef DEBUG
printf( "vips_get_disc_threshold: %zd bytes\n", threshold );
#endif /*DEBUG*/
}
return( threshold );
}
/* Check two images for compatibility: their geometries need to match.
*/
static gboolean
vips_foreign_load_iscompat( VipsImage *a, VipsImage *b )
{
if( a->Xsize != b->Xsize ||
a->Ysize != b->Ysize ||
a->Bands != b->Bands ||
a->Coding != b->Coding ||
a->BandFmt != b->BandFmt ) {
vips_error( "VipsForeignLoad",
"%s", _( "images do not match" ) );
return( FALSE );
}
return( TRUE );
}
/* Our start function ... do the lazy open, if necessary, and return a region
* on the new image.
*/
@ -658,7 +703,6 @@ static void *
vips_foreign_load_start( VipsImage *out, void *a, void *dummy )
{
VipsForeignLoad *load = VIPS_FOREIGN_LOAD( a );
VipsObjectClass *object_class = VIPS_OBJECT_GET_CLASS( a );
VipsForeignLoadClass *class = VIPS_FOREIGN_LOAD_GET_CLASS( a );
if( !load->real ) {
@ -691,8 +735,7 @@ vips_foreign_load_start( VipsImage *out, void *a, void *dummy )
printf( "vips_foreign_load_start: making 'p' temp\n" );
#endif /*DEBUG*/
if( !(load->real = vips_image_new()) )
return( NULL );
load->real = vips_image_new();
}
#ifdef DEBUG
@ -712,16 +755,8 @@ vips_foreign_load_start( VipsImage *out, void *a, void *dummy )
* Some versions of ImageMagick give different results between
* Ping and Load for some formats, for example.
*/
if( load->real->Xsize != out->Xsize ||
load->real->Ysize != out->Ysize ||
load->real->Bands != out->Bands ||
load->real->Coding != out->Coding ||
load->real->BandFmt != out->BandFmt ) {
vips_error( object_class->nickname,
"%s", _( "header() and load() report "
"different dimensions" ) );
if( !vips_foreign_load_iscompat( load->real, out ) )
return( NULL );
}
}
return( vips_region_new( load->real ) );
@ -750,6 +785,73 @@ vips_foreign_load_generate( VipsRegion *or,
return( 0 );
}
static int
vips_foreign_load_seq_generate( VipsRegion *or,
void *seq, void *a, void *b, gboolean *stop )
{
VipsForeignLoad *load = (VipsForeignLoad *) b;
VipsRegion *ir = (VipsRegion *) seq;
VipsRect *r = &or->valid;
/* The y pos of the request must be the same as our current file
* position.
*/
if( r->top != load->y_pos ) {
vips_error( "VipsForeignLoad",
_( "non-sequential read --- "
"at position %d in file, but position %d requested" ),
load->y_pos, r->top );
return( -1 );
}
/* We're inside a tilecache where tiles are the fill image width, so
* this should always be true.
*/
g_assert( r->left == 0 );
g_assert( r->width == or->im->Xsize );
g_assert( VIPS_RECT_BOTTOM( r ) <= or->im->Ysize );
/* Ask for input we need.
*/
if( vips_region_prepare( ir, r ) )
return( -1 );
/* Attach output region to that.
*/
if( vips_region_region( or, ir, r, r->left, r->top ) )
return( -1 );
load->y_pos += r->height;
return( 0 );
}
/* Like vips_copy(), but check sequentiality of access.
*/
static int
vips_foreign_load_seq( VipsForeignLoad *load, VipsImage *in, VipsImage **out )
{
VipsImage *seq;
seq = vips_image_new();
vips_demand_hint( seq, VIPS_DEMAND_STYLE_FATSTRIP,
in, NULL );
if( vips_image_pio_input( in ) ||
vips_image_copy_fields( seq, in ) ||
vips_image_generate( seq,
vips_start_one,
vips_foreign_load_seq_generate,
vips_stop_one,
in, load ) ) {
g_object_unref( seq );
return( -1 );
}
*out = seq;
return( 0 );
}
static int
vips_foreign_load_build( VipsObject *object )
{
@ -765,6 +867,15 @@ vips_foreign_load_build( VipsObject *object )
flags = 0;
if( class->get_flags )
flags |= class->get_flags( load );
if( (flags & VIPS_FOREIGN_PARTIAL) &&
(flags & VIPS_FOREIGN_SEQUENTIAL) ) {
vips_warn( "VipsForeign", "%s",
_( "VIPS_FOREIGN_PARTIAL and VIPS_FOREIGN_SEQUENTIAL "
"both set -- using SEQUENTIAL" ) );
flags ^= VIPS_FOREIGN_PARTIAL;
}
g_object_set( load, "flags", flags, NULL );
if( VIPS_OBJECT_CLASS( vips_foreign_load_parent_class )->
@ -784,12 +895,102 @@ vips_foreign_load_build( VipsObject *object )
return( -1 );
/* If there's no ->load() method then the header read has done
* everything. Otherwise, it's just set fields and we now must
* convert pixels on demand.
* everything. Otherwise, it's just set fields and we must also
* load pixels.
*
* Three modes:
*
* PARTIAL: just load and copy to @out. This is rather unlikely, most
* partial readers would just define a ->header() method I suppose.
*
* SEQUENTIAL: we need a tile cache plus a thing
* to check sequentiality.
*
* ELSE: it's a write-line thing. We delay the load until the
* first read, and allocate a memory/disk buffer as required.
*/
if( class->load ) {
if( class->load &&
(load->flags & VIPS_FOREIGN_PARTIAL) ) {
#ifdef DEBUG
printf( "vips_foreign_load_build: triggering ->load()\n" );
printf( "vips_foreign_load_build: partial read\n" );
#endif /*DEBUG*/
/* Read the image to @real.
*/
load->real = vips_image_new();
if( class->load( load ) ||
vips_image_pio_input( load->real ) )
return( -1 );
/* Must match ->header().
*/
if( !vips_foreign_load_iscompat( load->real, load->out ) )
return( -1 );
/* ->header() should set the dhint. It'll default to the safe
* SMALLTILE if ->header() did not set it.
*/
vips_demand_hint( load->out, load->out->dhint,
load->real, NULL );
if( vips_image_generate( load->out,
vips_start_one,
vips_foreign_load_generate,
vips_stop_one,
load->real, load ) )
return( -1 );
}
else if( class->load &&
(load->flags & VIPS_FOREIGN_SEQUENTIAL) ) {
VipsImage **t = (VipsImage **)
vips_object_local_array( VIPS_OBJECT( load ), 2 );
#ifdef DEBUG
printf( "vips_foreign_load_build: sequential read\n" );
#endif /*DEBUG*/
/* Load to @real.
*/
load->real = vips_image_new();
if( class->load( load ) ||
vips_image_pio_input( load->real ) )
return( -1 );
/* Must match ->header().
*/
if( !vips_foreign_load_iscompat( load->real, load->out ) )
return( -1 );
/* Copy to seq, checking sequentiality of accesses.
*/
if( vips_foreign_load_seq( load, load->real, &t[0] ) )
return( -1 );
/* Copy again, with a cache. Enough tiles for two complete
* rows.
*/
if( vips_tilecache( t[0], &t[1],
"tile_width", load->real->Xsize,
"tile_height", VIPS__TILE_HEIGHT,
"max_tiles", 2,
NULL ) )
return( -1 );
/* Finally, copy to out.
*/
vips_demand_hint( load->out, load->out->dhint, t[1], NULL );
if( vips_image_pio_input( load->real ) ||
vips_image_generate( load->out,
vips_start_one,
vips_foreign_load_generate,
vips_stop_one,
t[1], load ) )
return( -1 );
}
else if( class->load ) {
#ifdef DEBUG
printf( "vips_foreign_load_build: whole-image read\n" );
#endif /*DEBUG*/
/* ->header() should set the dhint. It'll default to the safe
@ -804,7 +1005,7 @@ vips_foreign_load_build( VipsObject *object )
vips_foreign_load_start,
vips_foreign_load_generate,
vips_stop_one,
load, NULL ) )
NULL, load ) )
return( -1 );
}
@ -846,6 +1047,13 @@ vips_foreign_load_class_init( VipsForeignLoadClass *class )
VIPS_ARGUMENT_OPTIONAL_INPUT,
G_STRUCT_OFFSET( VipsForeignLoad, disc ),
TRUE );
VIPS_ARG_BOOL( class, "sequential", 10,
_( "Sequential" ),
_( "Sequential read only" ),
VIPS_ARGUMENT_OPTIONAL_INPUT,
G_STRUCT_OFFSET( VipsForeignLoad, sequential ),
FALSE );
}
static void

View File

@ -58,10 +58,6 @@ typedef struct _VipsForeignLoadPng {
*/
char *filename;
/* Setting this means "I promise to only read sequentially from this
* image".
*/
gboolean sequential;
} VipsForeignLoadPng;
typedef VipsForeignLoadClass VipsForeignLoadPngClass;
@ -72,24 +68,23 @@ G_DEFINE_TYPE( VipsForeignLoadPng, vips_foreign_load_png,
static VipsForeignFlags
vips_foreign_load_png_get_flags_filename( const char *filename )
{
/* We can't tell just from the file. Always refuse.
*/
return( 0 );
VipsForeignFlags flags;
flags = 0;
if( vips__png_isinterlaced( filename ) )
flags = VIPS_FOREIGN_PARTIAL;
else
flags = VIPS_FOREIGN_SEQUENTIAL;
return( flags );
}
static VipsForeignFlags
vips_foreign_load_png_get_flags( VipsForeignLoad *load )
{
VipsForeignLoadPng *png = (VipsForeignLoadPng *) load;
VipsForeignFlags flags;
flags = 0;
/* If sequential is set, try to read partially.
*/
if( png->sequential )
flags |= VIPS_FOREIGN_PARTIAL;
return( flags );
return( vips_foreign_load_png_get_flags_filename( png->filename ) );
}
static int
@ -108,7 +103,7 @@ vips_foreign_load_png_load( VipsForeignLoad *load )
{
VipsForeignLoadPng *png = (VipsForeignLoadPng *) load;
if( vips__png_read( png->filename, load->real, png->sequential ) )
if( vips__png_read( png->filename, load->real ) )
return( -1 );
return( 0 );
@ -143,13 +138,6 @@ vips_foreign_load_png_class_init( VipsForeignLoadPngClass *class )
VIPS_ARGUMENT_REQUIRED_INPUT,
G_STRUCT_OFFSET( VipsForeignLoadPng, filename ),
NULL );
VIPS_ARG_BOOL( class, "sequential", 10,
_( "Sequential" ),
_( "Sequential read only" ),
VIPS_ARGUMENT_OPTIONAL_INPUT,
G_STRUCT_OFFSET( VipsForeignLoadPng, sequential ),
FALSE );
}
static void

View File

@ -113,11 +113,6 @@ typedef struct {
png_infop pInfo;
png_bytep *row_pointer;
png_bytep data;
/* During sequential read we keep track of the current y position in
* the file to check sequentiality.
*/
int y_pos;
} Read;
static void
@ -145,7 +140,6 @@ read_new( const char *name, VipsImage *out )
read->pInfo = NULL;
read->row_pointer = NULL;
read->data = NULL;
read->y_pos = 0;
g_signal_connect( out, "close",
G_CALLBACK( read_destroy ), read );
@ -342,52 +336,35 @@ png2vips_interlace( Read *read, VipsImage *out )
}
static int
png2vips_generate( VipsRegion *out,
png2vips_generate( VipsRegion *or,
void *seq, void *a, void *b, gboolean *stop )
{
VipsRect *r = &or->valid;
Read *read = (Read *) a;
int y;
#ifdef DEBUG
printf( "png2vips_generate: line %d, %d rows\n",
out->valid.top, out->valid.height );
printf( "png2vips_generate: thread %p\n", g_thread_self() );
r->top, r->height );
#endif /*DEBUG*/
/* The y pos of the request must be the same as our current file
* position.
*/
if( out->valid.top != read->y_pos ) {
vips_error( "png2vips", _( "non-sequential read --- "
"at position %d in file, but position %d requested" ),
read->y_pos, out->valid.top );
return( -1 );
}
if( setjmp( png_jmpbuf( read->pPng ) ) )
return( -1 );
/* We're inside a tilecache where tiles are the fill image width, so
* this should always be true.
*/
g_assert( out->valid.left == 0 );
g_assert( out->valid.width == out->im->Xsize );
g_assert( VIPS_RECT_BOTTOM( &out->valid ) <= out->im->Ysize );
g_assert( r->left == 0 );
g_assert( r->width == or->im->Xsize );
g_assert( VIPS_RECT_BOTTOM( r ) <= or->im->Ysize );
for( y = 0; y < out->valid.height; y++ ) {
png_bytep q = (png_bytep)
VIPS_REGION_ADDR( out, 0, out->valid.top + y );
for( y = 0; y < r->height; y++ ) {
png_bytep q = (png_bytep) VIPS_REGION_ADDR( or, 0, r->top + y );
png_read_row( read->pPng, q, NULL );
}
read->y_pos += out->valid.height;
#ifdef DEBUG
printf( "png2vips_generate: done for thread %p\n", g_thread_self() );
#endif /*DEBUG*/
return( 0 );
}
@ -396,7 +373,7 @@ png2vips_generate( VipsRegion *out,
*
* We are behind a tile cache, so we can assume that vips_image_generate()
* will always be asked for tiles which are the full image width. We can also
* assume we are single-threaded.
* assume we are single-threaded. And that we will be called sequentially.
*/
static int
png2vips_sequential( Read *read, VipsImage *out )
@ -409,34 +386,29 @@ png2vips_sequential( Read *read, VipsImage *out )
return( 0 );
}
/* Noninterlaced images can be read without needing enough RAM for the whole
* image.
/* Interlaced PNGs need to be entirely decompressed into memory then can be
* served partially from there. Non-interlaced PNGs may be read sequentially.
*/
static int
png2vips_noninterlace( Read *read, VipsImage *out )
gboolean
vips__png_isinterlaced( const char *filename )
{
const int rowbytes = VIPS_IMAGE_SIZEOF_LINE( out );
const int height = png_get_image_height( read->pPng, read->pInfo );
VipsImage *image;
Read *read;
gboolean interlaced;
int y;
if( !(read->data = (png_bytep) vips_malloc( NULL, rowbytes )) )
image = vips_image_new();
if( !(read = read_new( filename, image )) ) {
g_object_unref( image );
return( -1 );
if( setjmp( png_jmpbuf( read->pPng ) ) )
return( -1 );
for( y = 0; y < height; y++ ) {
png_read_row( read->pPng, read->data, NULL );
if( vips_image_write_line( out, y, read->data ) )
return( -1 );
}
interlaced = png_set_interlace_handling( read->pPng ) > 1;
g_object_unref( image );
return( 0 );
return( interlaced );
}
int
vips__png_read( const char *name, VipsImage *out, int sequential )
vips__png_read( const char *name, VipsImage *out )
{
Read *read;
@ -452,38 +424,9 @@ vips__png_read( const char *name, VipsImage *out, int sequential )
png2vips_interlace( read, out ) )
return( -1 );
}
else if( sequential ) {
VipsImage *raw;
VipsImage *t;
/* Read to this image, then cache to out.
*/
raw = vips_image_new();
vips_object_local( out, raw );
if( png2vips_header( read, raw ) ||
png2vips_sequential( read, raw ) )
return( -1 );
/* Copy to out, adding a cache. Enough tiles for two complete
* rows.
*/
if( vips_tilecache( raw, &t,
"tile_width", raw->Xsize,
"tile_height", VIPS__TILE_HEIGHT,
"max_tiles", 2,
NULL ) )
return( -1 );
if( vips_image_write( t, out ) ) {
g_object_unref( t );
return( -1 );
}
g_object_unref( t );
}
else {
if( png2vips_header( read, out ) ||
png2vips_noninterlace( read, out ) )
png2vips_sequential( read, out ) )
return( -1 );
}

View File

@ -35,8 +35,9 @@ extern "C" {
#endif /*__cplusplus*/
int vips__png_header( const char *name, VipsImage *out );
int vips__png_read( const char *name, VipsImage *out, int sequential );
int vips__png_read( const char *name, VipsImage *out );
int vips__png_ispng( const char *filename );
gboolean vips__png_isinterlaced( const char *filename );
extern const char *vips__png_suffs[];
int vips__png_write( VipsImage *in, const char *filename,

View File

@ -89,8 +89,9 @@ void *vips_foreign_map( const char *base,
typedef enum {
VIPS_FOREIGN_NONE = 0, /* No flags set */
VIPS_FOREIGN_PARTIAL = 1, /* Lazy read OK (eg. tiled tiff) */
VIPS_FOREIGN_BIGENDIAN = 2, /* Most-significant byte first */
VIPS_FOREIGN_ALL = 3 /* All flags set */
VIPS_FOREIGN_SEQUENTIAL = 2, /* Top-to-bottom lazy read OK */
VIPS_FOREIGN_BIGENDIAN = 3, /* Most-significant byte first */
VIPS_FOREIGN_ALL = 4 /* All flags set */
} VipsForeignFlags;
#define VIPS_TYPE_FOREIGN_LOAD (vips_foreign_load_get_type())
@ -116,10 +117,20 @@ typedef struct _VipsForeignLoad {
*/
gboolean disc;
/* Flags read from the foreign.
/* Setting this means "I promise to only read sequentially from this
* image".
*/
gboolean sequential;
/* Flags for this load operation.
*/
VipsForeignFlags flags;
/* In sequential mode we need to track the y position so we can
* ensure top-to-bottom-ness.
*/
int y_pos;
/*< public >*/
/* The image we generate. This must be set by ->header().
@ -130,7 +141,6 @@ typedef struct _VipsForeignLoad {
* disc foreign or a memory buffer. This must be set by ->load().
*/
VipsImage *real;
} VipsForeignLoad;
typedef struct _VipsForeignLoadClass {
@ -139,9 +149,8 @@ typedef struct _VipsForeignLoadClass {
/* Is a file in this format.
*
* This function should return %TRUE if
* the file contains an image of this type. If you don't define this
* function, #VipsForeignLoad
* This function should return %TRUE if the file contains an image of
* this type. If you don't define this function, #VipsForeignLoad
* will use @suffs instead.
*/
gboolean (*is_a)( const char * );
@ -156,13 +165,15 @@ typedef struct _VipsForeignLoadClass {
*/
VipsForeignFlags (*get_flags_filename)( const char * );
/* Get the flags for this image. Images can be loaded from (for
* example) memory areas rather than files, so you can't just use
/* Get the flags for this load operation. Images can be loaded from
* (for example) memory areas rather than files, so you can't just use
* @get_flags_filename().
*/
VipsForeignFlags (*get_flags)( VipsForeignLoad * );
/* Set the header fields in @out from @filename. If you can read the
/* Do the minimum read we can.
*
* Set the header fields in @out from @filename. If you can read the
* whole image as well with no performance cost (as with vipsload),
* or if your loader does not support reading only the header, read
* the entire image in this method and leave @load() NULL.
@ -175,7 +186,8 @@ typedef struct _VipsForeignLoadClass {
*/
int (*header)( VipsForeignLoad * );
/* Read the whole image into @real. It gets copied to @out later.
/* Read the whole image into @real. The pixels will get copied to @out
* later.
*
* You can omit this method if you define a @header() method which
* loads the whole file.

View File

@ -14,6 +14,7 @@ vips_foreign_flags_get_type( void )
static const GEnumValue values[] = {
{VIPS_FOREIGN_NONE, "VIPS_FOREIGN_NONE", "none"},
{VIPS_FOREIGN_PARTIAL, "VIPS_FOREIGN_PARTIAL", "partial"},
{VIPS_FOREIGN_SEQUENTIAL, "VIPS_FOREIGN_SEQUENTIAL", "sequential"},
{VIPS_FOREIGN_BIGENDIAN, "VIPS_FOREIGN_BIGENDIAN", "bigendian"},
{VIPS_FOREIGN_ALL, "VIPS_FOREIGN_ALL", "all"},
{0, NULL, NULL}

View File

@ -67,9 +67,9 @@
*/
/*
#define DEBUG_FATAL
#define DEBUG
*/
#define DEBUG_FATAL
/* Need to disable these sometimes.
#undef DEBUG_FATAL
@ -1010,6 +1010,7 @@ main( int argc, char **argv )
G_LOG_LEVEL_ERROR |
G_LOG_LEVEL_CRITICAL |
G_LOG_LEVEL_WARNING );
fprintf( stderr, "** DEBUG_FATAL\n" );
#endif /*!DEBUG_FATAL*/
/* Try to find our action.