free main-thread pixel buffers

threads keep pixel buffers in thread-private storage, and free these
buffers on thread exit ... this means buffers created by the main thread will
only be freed on program exit!

if your program creates any main-thread buffers, these buffers will
eventually fill the operation cache and force everything else out,
making the cache useless

this patch explicitly frees main-thread pixel buffers on image close

ChangeLog

@@ -14,6 +14,8 @@
 - tiffsave converts for jpg if jpg compression is turned on
 - tiffsave supports --strip
 - conversions to GREY16 could lock
+- main-thread pixel buffers are freed on image close, improving cache
+  behaviour
 
 18/5/16 started 8.3.2
 - more robust vips image reading

TODO

@@ -1,26 +1,3 @@
-- we free buffers on vips__buffer_shutdown(), called from thread shutdown,
-  but this won't happen for the main thread until program exit!
-
-  therefore any buffers allocated on the main thread will never leave cache,
-  and will eventually take over, forcing all operations out
-
-  we need to free buffers on image close, not thread exit
-
-  structure:
-
-	thread private ->
-		VipsBufferThread * -> hash(im) -> 
-			VipsBufferCache * -> 
-				array of VipsBuffer
-
-  we currently free VipsBufferThread on thread exit, instead we must free
-  VipsBufferCache on image close
-
-  alternative: keep the current thing, but keep a special note of the main 
-  thread VipsBufferCache ... on image close, remove any buffers just from that 
-
-  we'd need vips__thread_main to track the thread that called VIPS_INIT
-
 - add more webp tests to py suite
 
 - try moving some more of the CLI tests to py

libvips/include/vips/internal.h

@@ -102,6 +102,8 @@ extern char *vips__cache_max_files;
 extern gboolean vips__cache_dump;
 extern gboolean vips__cache_trace;
 
+extern GThread *vips__thread_main;
+
 void vips__cache_init( void );
 
 void vips__print_renders( void );

libvips/iofuncs/buffer.c

@@ -20,6 +20,8 @@
  * 18/12/13
  * 	- keep a few buffers in reserve per image, stops malloc/free 
  * 	  cycling when sharing is repeatedly discovered
+ * 5/6/16
+ * 	- free main thread buffers on image close
  */
 
 /*
@@ -214,6 +216,16 @@ buffer_cache_free( VipsBufferCache *cache )
 	g_free( cache );
 }
 
+static void
+buffer_cache_image_close_cb( VipsObject *object, void *data )
+{
+	VipsImage *im = VIPS_IMAGE( object ); 
+	VipsBufferCache *cache = (VipsBufferCache *) data;
+	VipsBufferThread *buffer_thread = cache->buffer_thread;
+
+	g_hash_table_remove( buffer_thread->hash, im );
+}
+
 static VipsBufferCache *
 buffer_cache_new( VipsBufferThread *buffer_thread, VipsImage *im )
 {
@@ -227,6 +239,16 @@ buffer_cache_new( VipsBufferThread *buffer_thread, VipsImage *im )
 	cache->reserve = NULL;
 	cache->n_reserve = 0;
 
+	/* The buffers on worker threads are freed on thread exit, but buffers
+	 * on the main thread won't be freed until program exit, and will
+	 * eventually fill the cache.
+	 *
+	 * If this is a main-thread buffer, junk it by hand on image close. 
+	 */
+	if( cache->thread == vips__thread_main ) 
+		g_signal_connect( im, "close", 
+			G_CALLBACK( buffer_cache_image_close_cb ), cache ); 
+
 #ifdef DEBUG_CREATE
 	g_mutex_lock( vips__global_lock );
 	vips__buffer_cache_all = 

libvips/iofuncs/init.c

@@ -94,6 +94,10 @@ int vips__fatal = 0;
  */
 GMutex *vips__global_lock = NULL;
 
+/* The thread that called vips_init().
+ */
+GThread *vips__thread_main = NULL;
+
 /* Keep a copy of the argv0 here.
  */
 static char *vips__argv0 = NULL;
@@ -140,6 +144,8 @@ vips_get_argv0( void )
  * must not call VIPS_INIT() after vips_shutdown(). In other words, you cannot
  * stop and restart vips. 
  *
+ * vips_shutdown() must be called from the same thread that called VIPS_INIT().
+ *
  * VIPS_INIT() does approximately the following:
  *
  * + checks that the libvips your program is expecting is 
@@ -154,7 +160,7 @@ vips_get_argv0( void )
  * + creates the main vips types, including #VipsImage and friends
  *
  * + loads any plugins from $libdir/vips-x.y/, where x and y are the
- *   major and minor version numbers for this VIPS.
+ *   major and minor version numbers for this VIPS
  *
  * + if your platform supports atexit(), VIPS_INIT() will ask for
  *   vips_shutdown() to be called on program exit
@@ -394,6 +400,11 @@ vips_init( const char *argv0 )
 	 */
 	vips_vector_init();
 
+	/* The main thread is never shut down, so we need to free some main
+	 * thread resources by hand. Track the maiin thread here. 
+	 */
+	vips__thread_main = g_thread_self();
+
 #ifdef HAVE_GSF
 	/* Use this for structured file write.
 	 */
@@ -506,6 +517,8 @@ vips_thread_shutdown( void )
  * You may call VIPS_INIT() many times and vips_shutdown() many times, but you 
  * must not call VIPS_INIT() after vips_shutdown(). In other words, you cannot
  * stop and restart vips. 
+ *
+ * vips_shutdown() must be called from the same thread that called VIPS_INIT().
  */
 void
 vips_shutdown( void )
@@ -514,6 +527,10 @@ vips_shutdown( void )
 	printf( "vips_shutdown:\n" );
 #endif /*DEBUG*/
 
+	/* Must be called by the main thread.
+	 */
+	g_assert( vips__thread_main == g_thread_self() );
+
 	vips_cache_drop_all();
 
 	im_close_plugins();