always use mmap windows

ChangeLog

@@ -2,6 +2,8 @@
 - branch for new dev cycle
 - argh, missing init from colour.c (thanks Peter)
 - argh, im_measure() was not looping over bands correctly (thanks Peter)
+- removed mmap_limit, we now always use windows ... reduces VM use hugely,
+  because mmap windows are freed when their regions are freed
 
 26/11/09 started 7.20.3
 - updated en_GB.po translation 

TODO

@@ -4,29 +4,16 @@ WONTFIX for 7.20
 - some of deprecated.h repeats stuff we do better in image.h, eg. addr()
   should just call IM_REGION_ADDR()
 
-- load 500 jpegs, group, shrink, save
-
-  VM use is huge, ouch, could we use a bit less?
-
-  maybe free all windows on evalend? are there regions left about? how come we
-  use so many resources?
-
-  get about 300 images in on the laptop
-
 - rename vipsCC in SWIG as pyvips?
 
 - im_flood*() should use inline rather than #define
 
 - try the new liboil thing:
 
-	http://www.schleef.org/orc/
+	http://code.entropywave.com/projects/orc/
 
   pick something like abs and time it
 
-- IMAGE->file_length should be gint64 rather than size_t?
-
-  if we make this change, need to remove extra casts from various uses
-
 - look into G_GNUC_DEPRECATED for back compat in vips8
 
 - use

libvips/include/vips/internal.h

@@ -123,7 +123,6 @@ void im__exr_register( void );
 void im__magick_register( void );
 
 extern int im__read_test;
-extern int im__mmap_limit;
 extern GMutex *im__global_lock;
 
 typedef enum {

libvips/include/vips/private.h

@@ -44,7 +44,6 @@ extern "C" {
 /* Private to iofuncs: the image size above which we switch from
  * mmap()-whole-image behaviour to mmap()-window, plus window margins.
  */
-#define IM__MMAP_LIMIT (1024*1024*30)
 #define IM__WINDOW_MARGIN (128)
 
 /* sizeof() a VIPS header on disc.
@@ -60,7 +59,7 @@ typedef enum {
 	IM_NONE,		/* no type set */
 	IM_SETBUF,		/* malloced memory array */
 	IM_SETBUF_FOREIGN,	/* memory array, don't free on close */
-	IM_OPENIN,		/* input from fd */
+	IM_OPENIN,		/* input from fd with a window */
 	IM_MMAPIN,		/* memory mapped input file */
 	IM_MMAPINRW,		/* memory mapped read/write file */
 	IM_OPENOUT,		/* output to fd */

libvips/iofuncs/check.c

@@ -225,7 +225,7 @@ im_incheck( IMAGE *im )
 		break;
 
 	case IM_OPENOUT:
-		/* Close file down and reopen as im_mmapin.
+		/* Close file down and reopen as input.
 		 */
 #ifdef DEBUG_IO
 		printf( "im_incheck: auto-rewind of %s\n", im->filename );

libvips/iofuncs/im_binfile.c

@@ -146,19 +146,6 @@ im_binfile( const char *name, int xsize, int ysize, int bands, int offset )
 		im_warn( "im_binfile", _( "%s is longer than expected" ),
 			im->filename );
 
-	/* If the predicted size is under our mmap threshold, mmap the whole
-	 * thing now. Otherwise, delay the map until region create and we'll
-	 * use a rolling window. See also im_openin().
-	 */
-	if( psize < im__mmap_limit ) {
-		if( im_mapfile( im ) ) {
-			im_close( im );
-			return( NULL );
-		}
-		im->data = im->baseaddr + offset;
-		im->dtype = IM_MMAPIN;
-	}
-
 	/* Set header fields.
 	 */
 	im->Xsize = xsize;

libvips/iofuncs/im_open_vips.c

@@ -9,6 +9,9 @@
  *	- fix signed/unsigned warnings
  * 12/10/09
  *	- heh argh reading history always stopped after the first line
+ * 9/12/09
+ * 	- only wholly map input files on im_incheck() ... this reduces VM use,
+ * 	  especially with large numbers of small files
  */
 
 /*
@@ -332,15 +335,6 @@
  */
 #define NAMESPACE "http://www.vips.ecs.soton.ac.uk/vips" 
 
-/* mmap() whole vs. window threshold ... an int, so we can tune easily from a
- * debugger.
- */
-#ifdef DEBUG
-int im__mmap_limit = 1;
-#else
-int im__mmap_limit = IM__MMAP_LIMIT;
-#endif /*DEBUG*/
-
 /* Sort of open for read for image files. Shared with im_binfile().
  */
 int
@@ -824,7 +818,7 @@ rebuild_header( IMAGE *im )
 	return( 0 );
 }
 
-/* Called at the end of im__read_header ... get any XML after the pixel data
+/* Called at the end of im_openin() ... get any XML after the pixel data
  * and read it in.
  */
 static int 
@@ -1127,8 +1121,8 @@ im__writehist( IMAGE *im )
 
 /* Open the filename, read the header, some sanity checking.
  */
-static int
+int
-im__read_header( IMAGE *image )
+im_openin( IMAGE *image )
 {
 	/* We don't use im->sizeof_header here, but we know we're reading a
 	 * VIPS image anyway.
@@ -1143,7 +1137,7 @@ im__read_header( IMAGE *image )
 		return( -1 );
 	if( read( image->fd, header, IM_SIZEOF_HEADER ) != IM_SIZEOF_HEADER ||
 		im__read_header_bytes( image, header ) ) {
-		im_error( "im_openin", 
+		im_error( "im__read_header", 
 			_( "unable to read header for \"%s\", %s" ),
 			image->filename, strerror( errno ) );
 		return( -1 );
@@ -1156,7 +1150,8 @@ im__read_header( IMAGE *image )
 		return( -1 );
 	image->file_length = rsize;
 	if( psize > rsize ) 
-		im_warn( "im_openin", _( "unable to read data for \"%s\", %s" ),
+		im_warn( "im__read_header", 
+			_( "unable to read data for \"%s\", %s" ),
 			image->filename, _( "file has been truncated" ) );
 
 	/* Set demand style. Allow the most permissive sort.
@@ -1168,7 +1163,7 @@ im__read_header( IMAGE *image )
 	 * harmless.
 	 */
 	if( im__readhist( image ) ) {
-		im_warn( "im_openin", _( "error reading XML: %s" ),
+		im_warn( "im__read_header", _( "error reading XML: %s" ),
 			im_error_buffer() );
 		im_error_clear();
 	}
@@ -1176,61 +1171,13 @@ im__read_header( IMAGE *image )
 	return( 0 );
 }
 
-/* Open, then mmap() small images, leave large images to have a rolling mmap()
- * window for each region. This is old and deprecated API, use im_vips_open()
- * in preference.
- */
-int
-im_openin( IMAGE *image )
-{
-	gint64 size;
-
-#ifdef DEBUG
-	char *str;
-
-	if( (str = g_getenv( "IM_MMAP_LIMIT" )) ) {
-		im__mmap_limit = atoi( str );
-		printf( "im_openin: setting maplimit to %d from environment\n",
-			im__mmap_limit );
-	}
-#endif /*DEBUG*/
-
-	if( im__read_header( image ) )
-		return( -1 );
-
-	/* Make sure we can map the whole thing without running over the VM
-	 * limit or running out of file.
-	 */
-	size = (gint64) IM_IMAGE_SIZEOF_LINE( image ) * image->Ysize + 
-		image->sizeof_header;
-	if( size < im__mmap_limit && image->file_length >= size ) {
-		if( im_mapfile( image ) )
-			return( -1 );
-		image->data = image->baseaddr + image->sizeof_header;
-		image->dtype = IM_MMAPIN;
-
-#ifdef DEBUG
-		printf( "im_openin: completely mmap()ing \"%s\": it's small\n",
-			image->filename );
-#endif /*DEBUG*/
-	}
-	else {
-#ifdef DEBUG
-		printf( "im_openin: delaying mmap() of \"%s\": it's big!\n",
-			image->filename );
-#endif /*DEBUG*/
-	}
-
-	return( 0 );
-}
-
 /* Open, then mmap() read/write. This is old and deprecated API, use
  * im_vips_open() in preference.
  */
 int
 im_openinrw( IMAGE *image )
 {
-	if( im__read_header( image ) )
+	if( im_openin( image ) )
 		return( -1 );
 	if( im_mapfilerw( image ) ) 
 		return( -1 );
@@ -1238,7 +1185,7 @@ im_openinrw( IMAGE *image )
 	image->dtype = IM_MMAPINRW;
 
 #ifdef DEBUG
-	printf( "im_openin: completely mmap()ing \"%s\" read-write\n",
+	printf( "im_openinrw: completely mmap()ing \"%s\" read-write\n",
 		image->filename );
 #endif /*DEBUG*/
 

libvips/iofuncs/window.c

@@ -33,8 +33,8 @@
  */
 
 /*
-#define DEBUG_TOTAL
 #define DEBUG_ENVIRONMENT
+#define DEBUG_TOTAL
 #define DEBUG
  */