sergiotarxz/libvips
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

always use mmap windows

Browse Source
This commit is contained in:
John Cupitt 2009-12-10 10:58:10 +00:00
parent 93323c44bb
commit 3ebc6d947f
8 changed files with 18 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ChangeLog
View File

@ -2,6 +2,8 @@
- branch for new dev cycle
- argh, missing init from colour.c (thanks Peter)
- argh, im_measure() was not looping over bands correctly (thanks Peter)
- removed mmap_limit, we now always use windows ... reduces VM use hugely,
because mmap windows are freed when their regions are freed
26/11/09 started 7.20.3
- updated en_GB.po translation

15
TODO
View File

@ -4,29 +4,16 @@ WONTFIX for 7.20
- some of deprecated.h repeats stuff we do better in image.h, eg. addr()
should just call IM_REGION_ADDR()
- load 500 jpegs, group, shrink, save
VM use is huge, ouch, could we use a bit less?
maybe free all windows on evalend? are there regions left about? how come we
use so many resources?
get about 300 images in on the laptop
- rename vipsCC in SWIG as pyvips?
- im_flood*() should use inline rather than #define
- try the new liboil thing:
http://www.schleef.org/orc/
http://code.entropywave.com/projects/orc/
pick something like abs and time it
- IMAGE->file_length should be gint64 rather than size_t?
if we make this change, need to remove extra casts from various uses
- look into G_GNUC_DEPRECATED for back compat in vips8
- use

1
libvips/include/vips/internal.h
View File

@ -123,7 +123,6 @@ void im__exr_register( void );
void im__magick_register( void );
extern int im__read_test;
extern int im__mmap_limit;
extern GMutex *im__global_lock;
typedef enum {

3
libvips/include/vips/private.h
View File

@ -44,7 +44,6 @@ extern "C" {
/* Private to iofuncs: the image size above which we switch from
* mmap()-whole-image behaviour to mmap()-window, plus window margins.
*/
#define IM__MMAP_LIMIT (1024*1024*30)
#define IM__WINDOW_MARGIN (128)
/* sizeof() a VIPS header on disc.
@ -60,7 +59,7 @@ typedef enum {
IM_NONE, /* no type set */
IM_SETBUF, /* malloced memory array */
IM_SETBUF_FOREIGN, /* memory array, don't free on close */
IM_OPENIN, /* input from fd */
IM_OPENIN, /* input from fd with a window */
IM_MMAPIN, /* memory mapped input file */
IM_MMAPINRW, /* memory mapped read/write file */
IM_OPENOUT, /* output to fd */

2
libvips/iofuncs/check.c
View File

@ -225,7 +225,7 @@ im_incheck( IMAGE *im )
break;
case IM_OPENOUT:
/* Close file down and reopen as im_mmapin.
/* Close file down and reopen as input.
*/
#ifdef DEBUG_IO
printf( "im_incheck: auto-rewind of %s\n", im->filename );

13
libvips/iofuncs/im_binfile.c
View File

@ -146,19 +146,6 @@ im_binfile( const char *name, int xsize, int ysize, int bands, int offset )
im_warn( "im_binfile", _( "%s is longer than expected" ),
im->filename );
/* If the predicted size is under our mmap threshold, mmap the whole
* thing now. Otherwise, delay the map until region create and we'll
* use a rolling window. See also im_openin().
*/
if( psize < im__mmap_limit ) {
if( im_mapfile( im ) ) {
im_close( im );
return( NULL );
}
im->data = im->baseaddr + offset;
im->dtype = IM_MMAPIN;
}
/* Set header fields.
*/
im->Xsize = xsize;

77
libvips/iofuncs/im_open_vips.c
View File

@ -9,6 +9,9 @@
* - fix signed/unsigned warnings
* 12/10/09
* - heh argh reading history always stopped after the first line
* 9/12/09
* - only wholly map input files on im_incheck() ... this reduces VM use,
* especially with large numbers of small files
*/
/*
@ -332,15 +335,6 @@
*/
#define NAMESPACE "http://www.vips.ecs.soton.ac.uk/vips"
/* mmap() whole vs. window threshold ... an int, so we can tune easily from a
* debugger.
*/
#ifdef DEBUG
int im__mmap_limit = 1;
#else
int im__mmap_limit = IM__MMAP_LIMIT;
#endif /*DEBUG*/
/* Sort of open for read for image files. Shared with im_binfile().
*/
int
@ -824,7 +818,7 @@ rebuild_header( IMAGE *im )
return( 0 );
}
/* Called at the end of im__read_header ... get any XML after the pixel data
/* Called at the end of im_openin() ... get any XML after the pixel data
* and read it in.
*/
static int
@ -1127,8 +1121,8 @@ im__writehist( IMAGE *im )
/* Open the filename, read the header, some sanity checking.
*/
static int
im__read_header( IMAGE *image )
int
im_openin( IMAGE *image )
{
/* We don't use im->sizeof_header here, but we know we're reading a
* VIPS image anyway.
@ -1143,7 +1137,7 @@ im__read_header( IMAGE *image )
return( -1 );
if( read( image->fd, header, IM_SIZEOF_HEADER ) != IM_SIZEOF_HEADER ||
im__read_header_bytes( image, header ) ) {
im_error( "im_openin",
im_error( "im__read_header",
_( "unable to read header for \"%s\", %s" ),
image->filename, strerror( errno ) );
return( -1 );
@ -1156,7 +1150,8 @@ im__read_header( IMAGE *image )
return( -1 );
image->file_length = rsize;
if( psize > rsize )
im_warn( "im_openin", _( "unable to read data for \"%s\", %s" ),
im_warn( "im__read_header",
_( "unable to read data for \"%s\", %s" ),
image->filename, _( "file has been truncated" ) );
/* Set demand style. Allow the most permissive sort.
@ -1168,7 +1163,7 @@ im__read_header( IMAGE *image )
* harmless.
*/
if( im__readhist( image ) ) {
im_warn( "im_openin", _( "error reading XML: %s" ),
im_warn( "im__read_header", _( "error reading XML: %s" ),
im_error_buffer() );
im_error_clear();
}
@ -1176,61 +1171,13 @@ im__read_header( IMAGE *image )
return( 0 );
}
/* Open, then mmap() small images, leave large images to have a rolling mmap()
* window for each region. This is old and deprecated API, use im_vips_open()
* in preference.
*/
int
im_openin( IMAGE *image )
{
gint64 size;
#ifdef DEBUG
char *str;
if( (str = g_getenv( "IM_MMAP_LIMIT" )) ) {
im__mmap_limit = atoi( str );
printf( "im_openin: setting maplimit to %d from environment\n",
im__mmap_limit );
}
#endif /*DEBUG*/
if( im__read_header( image ) )
return( -1 );
/* Make sure we can map the whole thing without running over the VM
* limit or running out of file.
*/
size = (gint64) IM_IMAGE_SIZEOF_LINE( image ) * image->Ysize +
image->sizeof_header;
if( size < im__mmap_limit && image->file_length >= size ) {
if( im_mapfile( image ) )
return( -1 );
image->data = image->baseaddr + image->sizeof_header;
image->dtype = IM_MMAPIN;
#ifdef DEBUG
printf( "im_openin: completely mmap()ing \"%s\": it's small\n",
image->filename );
#endif /*DEBUG*/
}
else {
#ifdef DEBUG
printf( "im_openin: delaying mmap() of \"%s\": it's big!\n",
image->filename );
#endif /*DEBUG*/
}
return( 0 );
}
/* Open, then mmap() read/write. This is old and deprecated API, use
* im_vips_open() in preference.
*/
int
im_openinrw( IMAGE *image )
{
if( im__read_header( image ) )
if( im_openin( image ) )
return( -1 );
if( im_mapfilerw( image ) )
return( -1 );
@ -1238,7 +1185,7 @@ im_openinrw( IMAGE *image )
image->dtype = IM_MMAPINRW;
#ifdef DEBUG
printf( "im_openin: completely mmap()ing \"%s\" read-write\n",
printf( "im_openinrw: completely mmap()ing \"%s\" read-write\n",
image->filename );
#endif /*DEBUG*/

2
libvips/iofuncs/window.c
View File

@ -33,8 +33,8 @@
*/
/*
#define DEBUG_TOTAL
#define DEBUG_ENVIRONMENT
#define DEBUG_TOTAL
#define DEBUG
*/