From 325f3b945145f80dfe25de25f94d684e24dceff6 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Sat, 19 Apr 2014 15:15:25 +0100 Subject: [PATCH] dzsave will write a zip file but see the TODO in dzsave.c - no way to pick a tile format, always jpeg - no way to pick a container format, always zip - needs testing - examine memuse, how well do we work for large zips - can we make a zip over 4gb? --- ChangeLog | 2 + configure.ac | 4 +- libvips/foreign/dzsave.c | 431 ++++++++++++++++++++++++++--------- libvips/include/vips/error.h | 1 + libvips/iofuncs/error.c | 25 +- libvips/iofuncs/init.c | 8 + libvips/iofuncs/threadpool.c | 6 +- 7 files changed, 364 insertions(+), 113 deletions(-) diff --git a/ChangeLog b/ChangeLog index fdfd72e4..1a282ac1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,8 @@ - vips_system() now supports many input images and you can change image argument order - support 16-bit palette TIFFs, plus palette TIFFs can have an alpha +- libgsf is now required +- dzsave can directly write a ZIP file 6/3/14 started 7.38.6 - grey ramp minimum was wrong diff --git a/configure.ac b/configure.ac index cbbe379d..be910efd 100644 --- a/configure.ac +++ b/configure.ac @@ -313,8 +313,8 @@ AC_CHECK_LIB(m,hypot,[AC_DEFINE(HAVE_HYPOT,1,[have hypot() in libm.])]) # have to have these # need glib 2.6 for GOption -PKG_CHECK_MODULES(REQUIRED, glib-2.0 >= 2.6 gmodule-2.0 libxml-2.0 gobject-2.0) -PACKAGES_USED="$PACKAGES_USED glib-2.0 libxml-2.0 gmodule-2.0 gobject-2.0" +PKG_CHECK_MODULES(REQUIRED, glib-2.0 >= 2.6 gmodule-2.0 libxml-2.0 gobject-2.0 libgsf-1) +PACKAGES_USED="$PACKAGES_USED glib-2.0 libxml-2.0 gmodule-2.0 gobject-2.0 libgsf-1" # after 2.28 we have a monotonic timer PKG_CHECK_MODULES(MONOTONIC, glib-2.0 >= 2.28,[ diff --git a/libvips/foreign/dzsave.c b/libvips/foreign/dzsave.c index 7f0ebeb0..dc4198b6 100644 --- a/libvips/foreign/dzsave.c +++ b/libvips/foreign/dzsave.c @@ -36,6 +36,31 @@ * - add --angle option * 19/6/13 * - faster --centre logic, thanks Kacey + * 18/4/14 + * - use libgsf for output so we can write to .zip etc. as well as the + * filesystem + */ + +/* + + TODO + + - need some way to pass save options to the jpeg tile writer + + - and to pick a tile writer! + + - need some way to select a container format + + - are the rules for naming objects correct? eg. + + vips dzsave fred.tif /my/output/file.zip + + will create file.zip containing + + file.zip:file_files + file.zip:file_files/0/0_0.jpg + file.zip:file.dzi + */ /* @@ -108,6 +133,8 @@ #include #include +#include + #include #include @@ -119,6 +146,161 @@ */ #define ROUND_UP(N,P) (ROUND_DOWN( (N) + (P) - 1, (P) )) +/* Simple wrapper around libgsf. + * + * We need to be able to do scattered writes to structred files. So while + * building a zip (for example) we need to be able to write to file/a/b.jpg, + * then to file/c/d.jpg, then back to file/a/e.jpg. This is tricky with the + * libgsf API which is happier doing writes in order. + * + * Put an API over libgsf to track refs to all directories and finish/close + * them. + */ + +/* Need to track the directory tree we are writing, with a ref for each + * GsfOutput. + */ +typedef struct _VipsGsfDirectory { + struct _VipsGsfDirectory *parent; + const char *name; + + /* List of child directories, if any. + */ + GSList *children; + + /* The GsfOutput we use for this object. + */ + GsfOutput *out; + +} VipsGsfDirectory; + +/* Close all dirs, non-NULL on error. + */ +static void * +vips_gsf_tree_close( VipsGsfDirectory *tree ) +{ + vips_slist_map2( tree->children, + (VipsSListMap2Fn) vips_gsf_tree_close, NULL, NULL ); + if( tree->out && + !gsf_output_is_closed( tree->out ) && + !gsf_output_close( tree->out ) ) { + vips_error( "vips_gsf", "%s", _( "unable to close stream" ) ); + return( tree ); + } + + return( NULL ); +} + +/* Close and unref everything, can't fail. Call vips_gsf_tree_close() to get + * an error return. + */ +static void * +vips_gsf_tree_free( VipsGsfDirectory *tree ) +{ + vips_slist_map2( tree->children, + (VipsSListMap2Fn) vips_gsf_tree_free, NULL, NULL ); + g_slist_free( tree->children ); + g_free( (char *) tree->name ); + if( tree->out ) { + if( !gsf_output_is_closed( tree->out ) ) + (void) gsf_output_close( tree->out ); + g_object_unref( tree->out ); + } + g_free( tree ); + + return( NULL ); +} + +/* Make a new tree root. + */ +static VipsGsfDirectory * +vips_gsf_tree_new( GsfOutput *out ) +{ + VipsGsfDirectory *tree = g_new( VipsGsfDirectory, 1 ); + + tree->parent = NULL; + tree->name = NULL; + tree->children = NULL; + tree->out = out; + + return( tree ); +} + +static void * +vips_gsf_child_by_name_sub( VipsGsfDirectory *dir, const char *name ) +{ + if( strcmp( dir->name, name ) == 0 ) + return( dir ); + + return( NULL ); +} + +/* Look up a child by name. + */ +static VipsGsfDirectory * +vips_gsf_child_by_name( VipsGsfDirectory *dir, const char *name ) +{ + return( vips_slist_map2( dir->children, + (VipsSListMap2Fn) vips_gsf_child_by_name_sub, + (char *) name, NULL ) ); +} + +/* Make a new directory. + */ +static VipsGsfDirectory * +vips_gsf_dir_new( VipsGsfDirectory *parent, const char *name ) +{ + VipsGsfDirectory *dir = g_new( VipsGsfDirectory, 1 ); + + g_assert( !vips_gsf_child_by_name( parent, name ) ); + + dir->parent = parent; + dir->name = g_strdup( name ); + dir->children = NULL; + dir->out = gsf_outfile_new_child_full( (GsfOutfile *) parent->out, + name, TRUE, + "compression-level", 0, + NULL ); + parent->children = g_slist_prepend( parent->children, dir ); + + return( dir ); +} + +/* Return a GsfOutput for writing to a path. Paths are object name first, then + * path components with least-specific first, NULL-terminated. For example: + * + * GsfOutput *obj = vips_gsf_path( tree, "fred.jpg", "a", "b", NULL ); + * + * Returns an obj you can use to write to a/b/fred.jpg. + * + * You must write, close and unref obj. + */ +static GsfOutput * +vips_gsf_path( VipsGsfDirectory *tree, const char *name, ... ) +{ + va_list ap; + VipsGsfDirectory *dir; + VipsGsfDirectory *child; + char *dir_name; + GsfOutput *obj; + + dir = tree; + va_start( ap, name ); + while( (dir_name = va_arg( ap, char * )) ) + if( (child = vips_gsf_child_by_name( dir, dir_name )) ) + dir = child; + else + dir = vips_gsf_dir_new( dir, dir_name ); + va_end( ap ); + + obj = gsf_outfile_new_child_full( (GsfOutfile *) dir->out, + name, FALSE, + "compression-level", 0, + NULL ); + + return( obj ); +} + typedef struct _VipsForeignSaveDz VipsForeignSaveDz; typedef struct _Layer Layer; @@ -189,6 +371,19 @@ struct _VipsForeignSaveDz { /* Count zoomify tiles we write. */ int tile_count; + + /* The tree structure we are writing tiles to. Can be filesystem, a + * zipfile, etc. + */ + VipsGsfDirectory *tree; + + /* @basename, but without a path at the start and without a suffix. + */ + char *name; + + /* The root directory name ... $(name)_files, $(name), etc. + */ + char *root_name; }; typedef VipsForeignSaveClass VipsForeignSaveDzClass; @@ -214,6 +409,9 @@ vips_foreign_save_dz_dispose( GObject *gobject ) VipsForeignSaveDz *dz = (VipsForeignSaveDz *) gobject; VIPS_FREEF( layer_free, dz->layer ); + VIPS_FREEF( vips_gsf_tree_free, dz->tree ); + VIPS_FREE( dz->name ); + VIPS_FREE( dz->root_name ); G_OBJECT_CLASS( vips_foreign_save_dz_parent_class )->dispose( gobject ); } @@ -358,56 +556,35 @@ pyramid_build( VipsForeignSaveDz *dz, Layer *above, return( layer ); } -static int -pyramid_mkdir( VipsForeignSaveDz *dz ) -{ - char buf[VIPS_PATH_MAX]; - - if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_DZ ) - vips_snprintf( buf, VIPS_PATH_MAX, "%s_files", dz->basename ); - else - vips_snprintf( buf, VIPS_PATH_MAX, "%s", dz->basename ); - - if( vips_existsf( "%s", buf ) ) { - vips_error( "dzsave", - _( "Directory \"%s\" exists" ), buf ); - return( -1 ); - } - if( vips_mkdirf( "%s", buf ) ) - return( -1 ); - - return( 0 ); -} - static int write_dzi( VipsForeignSaveDz *dz ) { - FILE *fp; + GsfOutput *out; char buf[VIPS_PATH_MAX]; char *p; - vips_snprintf( buf, VIPS_PATH_MAX, "%s.dzi", dz->basename ); - if( !(fp = vips__file_open_write( buf, TRUE )) ) - return( -1 ); + vips_snprintf( buf, VIPS_PATH_MAX, "%s.dzi", dz->name ); + out = vips_gsf_path( dz->tree, buf, NULL ); vips_snprintf( buf, VIPS_PATH_MAX, "%s", dz->suffix + 1 ); if( (p = (char *) vips__find_rightmost_brackets( buf )) ) *p = '\0'; - fprintf( fp, "\n" ); - fprintf( fp, "\n" ); + gsf_output_printf( out, "overlap ); - fprintf( fp, " TileSize=\"%d\"\n", dz->tile_size ); - fprintf( fp, " >\n" ); - fprintf( fp, " layer->height ); - fprintf( fp, " Width=\"%d\"\n", dz->layer->width ); - fprintf( fp, " />\n" ); - fprintf( fp, "\n" ); + gsf_output_printf( out, " Format=\"%s\"\n", buf ); + gsf_output_printf( out, " Overlap=\"%d\"\n", dz->overlap ); + gsf_output_printf( out, " TileSize=\"%d\"\n", dz->tile_size ); + gsf_output_printf( out, " >\n" ); + gsf_output_printf( out, " layer->height ); + gsf_output_printf( out, " Width=\"%d\"\n", dz->layer->width ); + gsf_output_printf( out, " />\n" ); + gsf_output_printf( out, "\n" ); - fclose( fp ); + (void) gsf_output_close( out ); + g_object_unref( out ); return( 0 ); } @@ -415,15 +592,12 @@ write_dzi( VipsForeignSaveDz *dz ) static int write_properties( VipsForeignSaveDz *dz ) { - FILE *fp; - char buf[VIPS_PATH_MAX]; + GsfOutput *out; - vips_snprintf( buf, VIPS_PATH_MAX, - "%s/ImageProperties.xml", dz->basename ); - if( !(fp = vips__file_open_write( buf, TRUE )) ) - return( -1 ); + out = vips_gsf_path( dz->tree, + "ImageProperties.xml", dz->root_name, NULL ); - fprintf( fp, "\n", dz->layer->width, @@ -431,7 +605,8 @@ write_properties( VipsForeignSaveDz *dz ) dz->tile_count, dz->tile_size ); - fclose( fp ); + (void) gsf_output_close( out ); + g_object_unref( out ); return( 0 ); } @@ -439,14 +614,15 @@ write_properties( VipsForeignSaveDz *dz ) static int write_blank( VipsForeignSaveDz *dz ) { - char buf[VIPS_PATH_MAX]; VipsImage *x, *t; int n; VipsArea *ones; double *d; int i; + void *buf; + size_t len; + GsfOutput *out; - vips_snprintf( buf, VIPS_PATH_MAX, "%s/blank.png", dz->basename ); if( vips_black( &x, dz->tile_size, dz->tile_size, NULL ) ) return( -1 ); @@ -468,12 +644,19 @@ write_blank( VipsForeignSaveDz *dz ) g_object_unref( x ); x = t; - if( vips_image_write_to_file( x, buf ) ) { + if( vips_pngsave_buffer( x, &buf, &len, NULL ) ) { g_object_unref( x ); return( -1 ); } g_object_unref( x ); + out = vips_gsf_path( dz->tree, "blank.png", dz->root_name, NULL ); + gsf_output_write( out, len, buf ); + gsf_output_close( out ); + g_object_unref( out ); + + g_free( buf ); + return( 0 ); } @@ -697,39 +880,28 @@ strip_allocate( VipsThreadState *state, void *a, gboolean *stop ) return( 0 ); } -/* Create a tile name in the current layout. +/* Make an output object for a tile in the current layout. */ -static int -tile_name( Layer *layer, char *buf, int x, int y ) +static GsfOutput * +tile_name( Layer *layer, int x, int y ) { - /* Need to lock around exists()/mkdir() or there's a race. - */ - static GMutex *file_lock = NULL; - static GOnce file_lock_once = G_ONCE_INIT; - VipsForeignSaveDz *dz = layer->dz; + GsfOutput *out; + char name[VIPS_PATH_MAX]; char dirname[VIPS_PATH_MAX]; char dirname2[VIPS_PATH_MAX]; Layer *p; int n; - file_lock = g_once( &file_lock_once, - (GThreadFunc) vips_g_mutex_new, NULL ); - g_mutex_lock( file_lock ); - switch( dz->layout ) { case VIPS_FOREIGN_DZ_LAYOUT_DZ: - vips_snprintf( dirname, VIPS_PATH_MAX, - "%s_files/%d", dz->basename, layer->n ); - vips_snprintf( buf, VIPS_PATH_MAX, - "%s/%d_%d%s", dirname, x, y, dz->suffix ); + vips_snprintf( dirname, VIPS_PATH_MAX, "%d", layer->n ); + vips_snprintf( name, VIPS_PATH_MAX, + "%d_%d%s", x, y, dz->suffix ); - if( !vips_existsf( "%s", dirname ) && - vips_mkdirf( "%s", dirname ) ) { - g_mutex_unlock( file_lock ); - return( -1 ); - } + out = vips_gsf_path( dz->tree, name, + dz->root_name, dirname, NULL ); break; @@ -750,54 +922,35 @@ tile_name( Layer *layer, char *buf, int x, int y ) */ n += y * layer->tiles_across + x; - vips_snprintf( dirname, VIPS_PATH_MAX, - "%s/TileGroup%d", dz->basename, n / 256 ); - vips_snprintf( buf, VIPS_PATH_MAX, - "%s/%d-%d-%d%s", dirname, layer->n, x, y, dz->suffix ); + vips_snprintf( dirname, VIPS_PATH_MAX, "TileGroup%d", n / 256 ); + vips_snprintf( name, VIPS_PATH_MAX, + "%d-%d-%d%s", layer->n, x, y, dz->suffix ); /* Used at the end in ImageProperties.xml */ dz->tile_count += 1; - if( !vips_existsf( "%s", dirname ) && - vips_mkdirf( "%s", dirname ) ) { - g_mutex_unlock( file_lock ); - return( -1 ); - } + out = vips_gsf_path( dz->tree, name, + dz->root_name, dirname, NULL ); break; case VIPS_FOREIGN_DZ_LAYOUT_GOOGLE: - vips_snprintf( dirname, VIPS_PATH_MAX, - "%s/%d", dz->basename, layer->n ); - vips_snprintf( dirname2, VIPS_PATH_MAX, - "%s/%d", dirname, y ); - vips_snprintf( buf, VIPS_PATH_MAX, - "%s/%d%s", dirname2, x, dz->suffix ); + vips_snprintf( dirname, VIPS_PATH_MAX, "%d", layer->n ); + vips_snprintf( dirname2, VIPS_PATH_MAX, "%d", y ); + vips_snprintf( name, VIPS_PATH_MAX, "%d%s", x, dz->suffix ); - if( !vips_existsf( "%s", dirname ) && - vips_mkdirf( "%s", dirname ) ) { - g_mutex_unlock( file_lock ); - return( -1 ); - } - - if( !vips_existsf( "%s", dirname2 ) && - vips_mkdirf( "%s", dirname2 ) ) { - g_mutex_unlock( file_lock ); - return( -1 ); - } + out = vips_gsf_path( dz->tree, name, + dz->root_name, dirname, dirname2, NULL ); break; default: g_assert( 0 ); - g_mutex_unlock( file_lock ); - return( -1 ); + return( NULL ); } - g_mutex_unlock( file_lock ); - - return( 0 ); + return( out ); } static int @@ -807,9 +960,11 @@ strip_work( VipsThreadState *state, void *a ) Layer *layer = strip->layer; VipsForeignSaveDz *dz = layer->dz; - char buf[VIPS_PATH_MAX]; VipsImage *x; VipsImage *t; + void *buf; + size_t len; + GsfOutput *out; #ifdef DEBUG_VERBOSE printf( "strip_work\n" ); @@ -837,10 +992,6 @@ strip_work( VipsThreadState *state, void *a ) } } - if( tile_name( layer, buf, - state->x / dz->tile_size, state->y / dz->tile_size ) ) - return( -1 ); - #ifdef DEBUG vips_object_sanity( VIPS_OBJECT( strip->image ) ); #endif /*DEBUG*/ @@ -870,11 +1021,34 @@ strip_work( VipsThreadState *state, void *a ) printf( "strip_work: writing to %s\n", buf ); #endif /*DEBUG_VERBOSE*/ + if( vips_jpegsave_buffer( x, &buf, &len, NULL ) ) { + g_object_unref( x ); + return( -1 ); + } + g_object_unref( x ); + + /* gsf doesn't like more than one write active at once. + */ + g_mutex_lock( vips__global_lock ); + + out = tile_name( layer, + state->x / dz->tile_size, state->y / dz->tile_size ); + + gsf_output_write( out, len, buf ); + gsf_output_close( out ); + g_object_unref( out ); + + g_mutex_unlock( vips__global_lock ); + + g_free( buf ); + + /* if( vips_image_write_to_file( x, buf ) ) { g_object_unref( x ); return( -1 ); } g_object_unref( x ); + */ #ifdef DEBUG_VERBOSE printf( "strip_work: success\n" ); @@ -1291,8 +1465,50 @@ vips_foreign_save_dz_build( VipsObject *object ) if( !(dz->layer = pyramid_build( dz, NULL, save->ready->Xsize, save->ready->Ysize, &real_pixels )) ) return( -1 ); - if( pyramid_mkdir( dz ) ) + + /* Make the thing we write the tiles into. + */ +{ + GsfOutput *out; + GsfOutput *zip; + GError *error = NULL; + + if( !(out = gsf_output_stdio_new( dz->basename, &error )) ) { + vips_g_error( &error ); return( -1 ); + } + + /* This is the zip we are building. + */ + if( !(zip = (GsfOutput *) gsf_outfile_zip_new( out, &error )) ) { + vips_g_error( &error ); + return( -1 ); + } + + /* We can unref @out since @zip has a ref to it. + */ + g_object_unref( out ); + + dz->tree = vips_gsf_tree_new( zip ); +} + + /* Drop any path stuff at the start of the output name and remove the + * suffix. + */ +{ + char *p; + + dz->name = g_path_get_basename( dz->basename ); + if( (p = (char *) vips__find_rightmost_brackets( dz->name )) ) + *p = '\0'; + if( (p = strrchr( dz->name, '.' )) ) + *p = '\0'; +} + + if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_DZ ) + dz->root_name = g_strdup_printf( "%s_files", dz->name ); + else + dz->root_name = g_strdup( dz->name ); if( vips_sink_disc( save->ready, pyramid_strip, dz ) ) return( -1 ); @@ -1318,6 +1534,9 @@ vips_foreign_save_dz_build( VipsObject *object ) return( -1 ); } + if( vips_gsf_tree_close( dz->tree ) ) + return( -1 ); + return( 0 ); } diff --git a/libvips/include/vips/error.h b/libvips/include/vips/error.h index 9bfdf139..ca8d8162 100644 --- a/libvips/include/vips/error.h +++ b/libvips/include/vips/error.h @@ -49,6 +49,7 @@ void vips_error_system( int err, const char *domain, const char *fmt, ... ) void vips_verror_system( int err, const char *domain, const char *fmt, va_list ap ); void vips_error_g( GError **error ); +void vips_g_error( GError **error ); void vips_warn( const char *domain, const char *fmt, ... ) __attribute__((format(printf, 2, 3))); void vips_vwarn( const char *domain, const char *fmt, va_list ap ); diff --git a/libvips/iofuncs/error.c b/libvips/iofuncs/error.c index 28cc8d99..ac35d22f 100644 --- a/libvips/iofuncs/error.c +++ b/libvips/iofuncs/error.c @@ -311,7 +311,9 @@ vips_error_system( int err, const char *domain, const char *fmt, ... ) * This function sets the glib error pointer from the vips error buffer and * clears it. It's handy for returning errors to glib functions from vips. * - * See also: g_set_error(). + * See vips_g_error() for the inverse operation. + * + * See also: g_set_error(), vips_g_error(). */ void vips_error_g( GError **error ) @@ -331,6 +333,27 @@ vips_error_g( GError **error ) vips_error_clear(); } + +/** + * vips_g_error: + * @error: glib error pointer + * + * This function adds the %GError to the vips error buffer and clears it. It's + * the opposite of vips_error_g(). + * + * See also: vips_error_g(). + */ +void +vips_g_error( GError **error ) +{ + if( error && + *error ) { + vips_error( "glib", "%s\n", (*error)->message ); + g_error_free( *error ); + *error = NULL; + } +} + /** * vips_error_clear: * diff --git a/libvips/iofuncs/init.c b/libvips/iofuncs/init.c index ced93528..c0f80d15 100644 --- a/libvips/iofuncs/init.c +++ b/libvips/iofuncs/init.c @@ -76,6 +76,8 @@ #include #include +#include + #include #include #include @@ -304,6 +306,10 @@ vips__init( const char *argv0 ) */ vips_vector_init(); + /* Use this for structured file write. + */ + gsf_init(); + /* Register vips_shutdown(). This may well not get called and many * platforms don't support it anyway. */ @@ -429,6 +435,8 @@ vips_shutdown( void ) vips__thread_profile_stop(); + gsf_shutdown(); + /* In dev releases, always show leaks. But not more than once, it's * annoying. */ diff --git a/libvips/iofuncs/threadpool.c b/libvips/iofuncs/threadpool.c index e17d05af..4dadc89a 100644 --- a/libvips/iofuncs/threadpool.c +++ b/libvips/iofuncs/threadpool.c @@ -200,10 +200,8 @@ vips_g_thread_new( const char *domain, GThreadFunc func, gpointer data ) #endif if( !thread ) { - if( error ) { - vips_error( domain, "%s", error->message ); - g_error_free( error ); - } + if( error ) + vips_g_error( &error ); else vips_error( domain, "%s", _( "unable to create thread" ) );