From 3ebc6d947fd1cc42e0cb147169bcc4d8e8a2990c Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Thu, 10 Dec 2009 10:58:10 +0000 Subject: [PATCH] always use mmap windows --- ChangeLog | 2 + TODO | 15 +------ libvips/include/vips/internal.h | 1 - libvips/include/vips/private.h | 3 +- libvips/iofuncs/check.c | 2 +- libvips/iofuncs/im_binfile.c | 13 ------ libvips/iofuncs/im_open_vips.c | 77 +++++---------------------------- libvips/iofuncs/window.c | 2 +- 8 files changed, 18 insertions(+), 97 deletions(-) diff --git a/ChangeLog b/ChangeLog index b55914d2..63d07212 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,8 @@ - branch for new dev cycle - argh, missing init from colour.c (thanks Peter) - argh, im_measure() was not looping over bands correctly (thanks Peter) +- removed mmap_limit, we now always use windows ... reduces VM use hugely, + because mmap windows are freed when their regions are freed 26/11/09 started 7.20.3 - updated en_GB.po translation diff --git a/TODO b/TODO index f2de0060..bf627243 100644 --- a/TODO +++ b/TODO @@ -4,29 +4,16 @@ WONTFIX for 7.20 - some of deprecated.h repeats stuff we do better in image.h, eg. addr() should just call IM_REGION_ADDR() -- load 500 jpegs, group, shrink, save - - VM use is huge, ouch, could we use a bit less? - - maybe free all windows on evalend? are there regions left about? how come we - use so many resources? - - get about 300 images in on the laptop - - rename vipsCC in SWIG as pyvips? - im_flood*() should use inline rather than #define - try the new liboil thing: - http://www.schleef.org/orc/ + http://code.entropywave.com/projects/orc/ pick something like abs and time it -- IMAGE->file_length should be gint64 rather than size_t? - - if we make this change, need to remove extra casts from various uses - - look into G_GNUC_DEPRECATED for back compat in vips8 - use diff --git a/libvips/include/vips/internal.h b/libvips/include/vips/internal.h index c73103ed..3c4674ae 100644 --- a/libvips/include/vips/internal.h +++ b/libvips/include/vips/internal.h @@ -123,7 +123,6 @@ void im__exr_register( void ); void im__magick_register( void ); extern int im__read_test; -extern int im__mmap_limit; extern GMutex *im__global_lock; typedef enum { diff --git a/libvips/include/vips/private.h b/libvips/include/vips/private.h index 62489791..e1098fb5 100644 --- a/libvips/include/vips/private.h +++ b/libvips/include/vips/private.h @@ -44,7 +44,6 @@ extern "C" { /* Private to iofuncs: the image size above which we switch from * mmap()-whole-image behaviour to mmap()-window, plus window margins. */ -#define IM__MMAP_LIMIT (1024*1024*30) #define IM__WINDOW_MARGIN (128) /* sizeof() a VIPS header on disc. @@ -60,7 +59,7 @@ typedef enum { IM_NONE, /* no type set */ IM_SETBUF, /* malloced memory array */ IM_SETBUF_FOREIGN, /* memory array, don't free on close */ - IM_OPENIN, /* input from fd */ + IM_OPENIN, /* input from fd with a window */ IM_MMAPIN, /* memory mapped input file */ IM_MMAPINRW, /* memory mapped read/write file */ IM_OPENOUT, /* output to fd */ diff --git a/libvips/iofuncs/check.c b/libvips/iofuncs/check.c index 6814ee28..acf7ef9b 100644 --- a/libvips/iofuncs/check.c +++ b/libvips/iofuncs/check.c @@ -225,7 +225,7 @@ im_incheck( IMAGE *im ) break; case IM_OPENOUT: - /* Close file down and reopen as im_mmapin. + /* Close file down and reopen as input. */ #ifdef DEBUG_IO printf( "im_incheck: auto-rewind of %s\n", im->filename ); diff --git a/libvips/iofuncs/im_binfile.c b/libvips/iofuncs/im_binfile.c index bab0d631..8c1b203a 100644 --- a/libvips/iofuncs/im_binfile.c +++ b/libvips/iofuncs/im_binfile.c @@ -146,19 +146,6 @@ im_binfile( const char *name, int xsize, int ysize, int bands, int offset ) im_warn( "im_binfile", _( "%s is longer than expected" ), im->filename ); - /* If the predicted size is under our mmap threshold, mmap the whole - * thing now. Otherwise, delay the map until region create and we'll - * use a rolling window. See also im_openin(). - */ - if( psize < im__mmap_limit ) { - if( im_mapfile( im ) ) { - im_close( im ); - return( NULL ); - } - im->data = im->baseaddr + offset; - im->dtype = IM_MMAPIN; - } - /* Set header fields. */ im->Xsize = xsize; diff --git a/libvips/iofuncs/im_open_vips.c b/libvips/iofuncs/im_open_vips.c index 3b0e4731..c665105c 100644 --- a/libvips/iofuncs/im_open_vips.c +++ b/libvips/iofuncs/im_open_vips.c @@ -9,6 +9,9 @@ * - fix signed/unsigned warnings * 12/10/09 * - heh argh reading history always stopped after the first line + * 9/12/09 + * - only wholly map input files on im_incheck() ... this reduces VM use, + * especially with large numbers of small files */ /* @@ -332,15 +335,6 @@ */ #define NAMESPACE "http://www.vips.ecs.soton.ac.uk/vips" -/* mmap() whole vs. window threshold ... an int, so we can tune easily from a - * debugger. - */ -#ifdef DEBUG -int im__mmap_limit = 1; -#else -int im__mmap_limit = IM__MMAP_LIMIT; -#endif /*DEBUG*/ - /* Sort of open for read for image files. Shared with im_binfile(). */ int @@ -824,7 +818,7 @@ rebuild_header( IMAGE *im ) return( 0 ); } -/* Called at the end of im__read_header ... get any XML after the pixel data +/* Called at the end of im_openin() ... get any XML after the pixel data * and read it in. */ static int @@ -1127,8 +1121,8 @@ im__writehist( IMAGE *im ) /* Open the filename, read the header, some sanity checking. */ -static int -im__read_header( IMAGE *image ) +int +im_openin( IMAGE *image ) { /* We don't use im->sizeof_header here, but we know we're reading a * VIPS image anyway. @@ -1143,7 +1137,7 @@ im__read_header( IMAGE *image ) return( -1 ); if( read( image->fd, header, IM_SIZEOF_HEADER ) != IM_SIZEOF_HEADER || im__read_header_bytes( image, header ) ) { - im_error( "im_openin", + im_error( "im__read_header", _( "unable to read header for \"%s\", %s" ), image->filename, strerror( errno ) ); return( -1 ); @@ -1156,7 +1150,8 @@ im__read_header( IMAGE *image ) return( -1 ); image->file_length = rsize; if( psize > rsize ) - im_warn( "im_openin", _( "unable to read data for \"%s\", %s" ), + im_warn( "im__read_header", + _( "unable to read data for \"%s\", %s" ), image->filename, _( "file has been truncated" ) ); /* Set demand style. Allow the most permissive sort. @@ -1168,7 +1163,7 @@ im__read_header( IMAGE *image ) * harmless. */ if( im__readhist( image ) ) { - im_warn( "im_openin", _( "error reading XML: %s" ), + im_warn( "im__read_header", _( "error reading XML: %s" ), im_error_buffer() ); im_error_clear(); } @@ -1176,61 +1171,13 @@ im__read_header( IMAGE *image ) return( 0 ); } -/* Open, then mmap() small images, leave large images to have a rolling mmap() - * window for each region. This is old and deprecated API, use im_vips_open() - * in preference. - */ -int -im_openin( IMAGE *image ) -{ - gint64 size; - -#ifdef DEBUG - char *str; - - if( (str = g_getenv( "IM_MMAP_LIMIT" )) ) { - im__mmap_limit = atoi( str ); - printf( "im_openin: setting maplimit to %d from environment\n", - im__mmap_limit ); - } -#endif /*DEBUG*/ - - if( im__read_header( image ) ) - return( -1 ); - - /* Make sure we can map the whole thing without running over the VM - * limit or running out of file. - */ - size = (gint64) IM_IMAGE_SIZEOF_LINE( image ) * image->Ysize + - image->sizeof_header; - if( size < im__mmap_limit && image->file_length >= size ) { - if( im_mapfile( image ) ) - return( -1 ); - image->data = image->baseaddr + image->sizeof_header; - image->dtype = IM_MMAPIN; - -#ifdef DEBUG - printf( "im_openin: completely mmap()ing \"%s\": it's small\n", - image->filename ); -#endif /*DEBUG*/ - } - else { -#ifdef DEBUG - printf( "im_openin: delaying mmap() of \"%s\": it's big!\n", - image->filename ); -#endif /*DEBUG*/ - } - - return( 0 ); -} - /* Open, then mmap() read/write. This is old and deprecated API, use * im_vips_open() in preference. */ int im_openinrw( IMAGE *image ) { - if( im__read_header( image ) ) + if( im_openin( image ) ) return( -1 ); if( im_mapfilerw( image ) ) return( -1 ); @@ -1238,7 +1185,7 @@ im_openinrw( IMAGE *image ) image->dtype = IM_MMAPINRW; #ifdef DEBUG - printf( "im_openin: completely mmap()ing \"%s\" read-write\n", + printf( "im_openinrw: completely mmap()ing \"%s\" read-write\n", image->filename ); #endif /*DEBUG*/ diff --git a/libvips/iofuncs/window.c b/libvips/iofuncs/window.c index eb036774..69947537 100644 --- a/libvips/iofuncs/window.c +++ b/libvips/iofuncs/window.c @@ -33,8 +33,8 @@ */ /* -#define DEBUG_TOTAL #define DEBUG_ENVIRONMENT +#define DEBUG_TOTAL #define DEBUG */