dzsave will write a zip file

but see the TODO in dzsave.c

- no way to pick a tile format, always jpeg
- no way to pick a container format, always zip
- needs testing
- examine memuse, how well do we work for large zips
- can we make a zip over 4gb?
This commit is contained in:
John Cupitt 2014-04-19 15:15:25 +01:00
parent 21f18ce0d2
commit 325f3b9451
7 changed files with 364 additions and 113 deletions

View File

@ -21,6 +21,8 @@
- vips_system() now supports many input images and you can change image
argument order
- support 16-bit palette TIFFs, plus palette TIFFs can have an alpha
- libgsf is now required
- dzsave can directly write a ZIP file
6/3/14 started 7.38.6
- grey ramp minimum was wrong

View File

@ -313,8 +313,8 @@ AC_CHECK_LIB(m,hypot,[AC_DEFINE(HAVE_HYPOT,1,[have hypot() in libm.])])
# have to have these
# need glib 2.6 for GOption
PKG_CHECK_MODULES(REQUIRED, glib-2.0 >= 2.6 gmodule-2.0 libxml-2.0 gobject-2.0)
PACKAGES_USED="$PACKAGES_USED glib-2.0 libxml-2.0 gmodule-2.0 gobject-2.0"
PKG_CHECK_MODULES(REQUIRED, glib-2.0 >= 2.6 gmodule-2.0 libxml-2.0 gobject-2.0 libgsf-1)
PACKAGES_USED="$PACKAGES_USED glib-2.0 libxml-2.0 gmodule-2.0 gobject-2.0 libgsf-1"
# after 2.28 we have a monotonic timer
PKG_CHECK_MODULES(MONOTONIC, glib-2.0 >= 2.28,[

View File

@ -36,6 +36,31 @@
* - add --angle option
* 19/6/13
* - faster --centre logic, thanks Kacey
* 18/4/14
* - use libgsf for output so we can write to .zip etc. as well as the
* filesystem
*/
/*
TODO
- need some way to pass save options to the jpeg tile writer
- and to pick a tile writer!
- need some way to select a container format
- are the rules for naming objects correct? eg.
vips dzsave fred.tif /my/output/file.zip
will create file.zip containing
file.zip:file_files
file.zip:file_files/0/0_0.jpg
file.zip:file.dzi
*/
/*
@ -108,6 +133,8 @@
#include <stdlib.h>
#include <string.h>
#include <gsf/gsf.h>
#include <vips/vips.h>
#include <vips/internal.h>
@ -119,6 +146,161 @@
*/
#define ROUND_UP(N,P) (ROUND_DOWN( (N) + (P) - 1, (P) ))
/* Simple wrapper around libgsf.
*
* We need to be able to do scattered writes to structred files. So while
* building a zip (for example) we need to be able to write to file/a/b.jpg,
* then to file/c/d.jpg, then back to file/a/e.jpg. This is tricky with the
* libgsf API which is happier doing writes in order.
*
* Put an API over libgsf to track refs to all directories and finish/close
* them.
*/
/* Need to track the directory tree we are writing, with a ref for each
* GsfOutput.
*/
typedef struct _VipsGsfDirectory {
struct _VipsGsfDirectory *parent;
const char *name;
/* List of child directories, if any.
*/
GSList *children;
/* The GsfOutput we use for this object.
*/
GsfOutput *out;
} VipsGsfDirectory;
/* Close all dirs, non-NULL on error.
*/
static void *
vips_gsf_tree_close( VipsGsfDirectory *tree )
{
vips_slist_map2( tree->children,
(VipsSListMap2Fn) vips_gsf_tree_close, NULL, NULL );
if( tree->out &&
!gsf_output_is_closed( tree->out ) &&
!gsf_output_close( tree->out ) ) {
vips_error( "vips_gsf", "%s", _( "unable to close stream" ) );
return( tree );
}
return( NULL );
}
/* Close and unref everything, can't fail. Call vips_gsf_tree_close() to get
* an error return.
*/
static void *
vips_gsf_tree_free( VipsGsfDirectory *tree )
{
vips_slist_map2( tree->children,
(VipsSListMap2Fn) vips_gsf_tree_free, NULL, NULL );
g_slist_free( tree->children );
g_free( (char *) tree->name );
if( tree->out ) {
if( !gsf_output_is_closed( tree->out ) )
(void) gsf_output_close( tree->out );
g_object_unref( tree->out );
}
g_free( tree );
return( NULL );
}
/* Make a new tree root.
*/
static VipsGsfDirectory *
vips_gsf_tree_new( GsfOutput *out )
{
VipsGsfDirectory *tree = g_new( VipsGsfDirectory, 1 );
tree->parent = NULL;
tree->name = NULL;
tree->children = NULL;
tree->out = out;
return( tree );
}
static void *
vips_gsf_child_by_name_sub( VipsGsfDirectory *dir, const char *name )
{
if( strcmp( dir->name, name ) == 0 )
return( dir );
return( NULL );
}
/* Look up a child by name.
*/
static VipsGsfDirectory *
vips_gsf_child_by_name( VipsGsfDirectory *dir, const char *name )
{
return( vips_slist_map2( dir->children,
(VipsSListMap2Fn) vips_gsf_child_by_name_sub,
(char *) name, NULL ) );
}
/* Make a new directory.
*/
static VipsGsfDirectory *
vips_gsf_dir_new( VipsGsfDirectory *parent, const char *name )
{
VipsGsfDirectory *dir = g_new( VipsGsfDirectory, 1 );
g_assert( !vips_gsf_child_by_name( parent, name ) );
dir->parent = parent;
dir->name = g_strdup( name );
dir->children = NULL;
dir->out = gsf_outfile_new_child_full( (GsfOutfile *) parent->out,
name, TRUE,
"compression-level", 0,
NULL );
parent->children = g_slist_prepend( parent->children, dir );
return( dir );
}
/* Return a GsfOutput for writing to a path. Paths are object name first, then
* path components with least-specific first, NULL-terminated. For example:
*
* GsfOutput *obj = vips_gsf_path( tree, "fred.jpg", "a", "b", NULL );
*
* Returns an obj you can use to write to a/b/fred.jpg.
*
* You must write, close and unref obj.
*/
static GsfOutput *
vips_gsf_path( VipsGsfDirectory *tree, const char *name, ... )
{
va_list ap;
VipsGsfDirectory *dir;
VipsGsfDirectory *child;
char *dir_name;
GsfOutput *obj;
dir = tree;
va_start( ap, name );
while( (dir_name = va_arg( ap, char * )) )
if( (child = vips_gsf_child_by_name( dir, dir_name )) )
dir = child;
else
dir = vips_gsf_dir_new( dir, dir_name );
va_end( ap );
obj = gsf_outfile_new_child_full( (GsfOutfile *) dir->out,
name, FALSE,
"compression-level", 0,
NULL );
return( obj );
}
typedef struct _VipsForeignSaveDz VipsForeignSaveDz;
typedef struct _Layer Layer;
@ -189,6 +371,19 @@ struct _VipsForeignSaveDz {
/* Count zoomify tiles we write.
*/
int tile_count;
/* The tree structure we are writing tiles to. Can be filesystem, a
* zipfile, etc.
*/
VipsGsfDirectory *tree;
/* @basename, but without a path at the start and without a suffix.
*/
char *name;
/* The root directory name ... $(name)_files, $(name), etc.
*/
char *root_name;
};
typedef VipsForeignSaveClass VipsForeignSaveDzClass;
@ -214,6 +409,9 @@ vips_foreign_save_dz_dispose( GObject *gobject )
VipsForeignSaveDz *dz = (VipsForeignSaveDz *) gobject;
VIPS_FREEF( layer_free, dz->layer );
VIPS_FREEF( vips_gsf_tree_free, dz->tree );
VIPS_FREE( dz->name );
VIPS_FREE( dz->root_name );
G_OBJECT_CLASS( vips_foreign_save_dz_parent_class )->dispose( gobject );
}
@ -358,56 +556,35 @@ pyramid_build( VipsForeignSaveDz *dz, Layer *above,
return( layer );
}
static int
pyramid_mkdir( VipsForeignSaveDz *dz )
{
char buf[VIPS_PATH_MAX];
if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_DZ )
vips_snprintf( buf, VIPS_PATH_MAX, "%s_files", dz->basename );
else
vips_snprintf( buf, VIPS_PATH_MAX, "%s", dz->basename );
if( vips_existsf( "%s", buf ) ) {
vips_error( "dzsave",
_( "Directory \"%s\" exists" ), buf );
return( -1 );
}
if( vips_mkdirf( "%s", buf ) )
return( -1 );
return( 0 );
}
static int
write_dzi( VipsForeignSaveDz *dz )
{
FILE *fp;
GsfOutput *out;
char buf[VIPS_PATH_MAX];
char *p;
vips_snprintf( buf, VIPS_PATH_MAX, "%s.dzi", dz->basename );
if( !(fp = vips__file_open_write( buf, TRUE )) )
return( -1 );
vips_snprintf( buf, VIPS_PATH_MAX, "%s.dzi", dz->name );
out = vips_gsf_path( dz->tree, buf, NULL );
vips_snprintf( buf, VIPS_PATH_MAX, "%s", dz->suffix + 1 );
if( (p = (char *) vips__find_rightmost_brackets( buf )) )
*p = '\0';
fprintf( fp, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" );
fprintf( fp, "<Image "
gsf_output_printf( out, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" );
gsf_output_printf( out, "<Image "
"xmlns=\"http://schemas.microsoft.com/deepzoom/2008\"\n" );
fprintf( fp, " Format=\"%s\"\n", buf );
fprintf( fp, " Overlap=\"%d\"\n", dz->overlap );
fprintf( fp, " TileSize=\"%d\"\n", dz->tile_size );
fprintf( fp, " >\n" );
fprintf( fp, " <Size \n" );
fprintf( fp, " Height=\"%d\"\n", dz->layer->height );
fprintf( fp, " Width=\"%d\"\n", dz->layer->width );
fprintf( fp, " />\n" );
fprintf( fp, "</Image>\n" );
gsf_output_printf( out, " Format=\"%s\"\n", buf );
gsf_output_printf( out, " Overlap=\"%d\"\n", dz->overlap );
gsf_output_printf( out, " TileSize=\"%d\"\n", dz->tile_size );
gsf_output_printf( out, " >\n" );
gsf_output_printf( out, " <Size \n" );
gsf_output_printf( out, " Height=\"%d\"\n", dz->layer->height );
gsf_output_printf( out, " Width=\"%d\"\n", dz->layer->width );
gsf_output_printf( out, " />\n" );
gsf_output_printf( out, "</Image>\n" );
fclose( fp );
(void) gsf_output_close( out );
g_object_unref( out );
return( 0 );
}
@ -415,15 +592,12 @@ write_dzi( VipsForeignSaveDz *dz )
static int
write_properties( VipsForeignSaveDz *dz )
{
FILE *fp;
char buf[VIPS_PATH_MAX];
GsfOutput *out;
vips_snprintf( buf, VIPS_PATH_MAX,
"%s/ImageProperties.xml", dz->basename );
if( !(fp = vips__file_open_write( buf, TRUE )) )
return( -1 );
out = vips_gsf_path( dz->tree,
"ImageProperties.xml", dz->root_name, NULL );
fprintf( fp, "<IMAGE_PROPERTIES "
gsf_output_printf( out, "<IMAGE_PROPERTIES "
"WIDTH=\"%d\" HEIGHT=\"%d\" NUMTILES=\"%d\" "
"NUMIMAGES=\"1\" VERSION=\"1.8\" TILESIZE=\"%d\" />\n",
dz->layer->width,
@ -431,7 +605,8 @@ write_properties( VipsForeignSaveDz *dz )
dz->tile_count,
dz->tile_size );
fclose( fp );
(void) gsf_output_close( out );
g_object_unref( out );
return( 0 );
}
@ -439,14 +614,15 @@ write_properties( VipsForeignSaveDz *dz )
static int
write_blank( VipsForeignSaveDz *dz )
{
char buf[VIPS_PATH_MAX];
VipsImage *x, *t;
int n;
VipsArea *ones;
double *d;
int i;
void *buf;
size_t len;
GsfOutput *out;
vips_snprintf( buf, VIPS_PATH_MAX, "%s/blank.png", dz->basename );
if( vips_black( &x, dz->tile_size, dz->tile_size, NULL ) )
return( -1 );
@ -468,12 +644,19 @@ write_blank( VipsForeignSaveDz *dz )
g_object_unref( x );
x = t;
if( vips_image_write_to_file( x, buf ) ) {
if( vips_pngsave_buffer( x, &buf, &len, NULL ) ) {
g_object_unref( x );
return( -1 );
}
g_object_unref( x );
out = vips_gsf_path( dz->tree, "blank.png", dz->root_name, NULL );
gsf_output_write( out, len, buf );
gsf_output_close( out );
g_object_unref( out );
g_free( buf );
return( 0 );
}
@ -697,39 +880,28 @@ strip_allocate( VipsThreadState *state, void *a, gboolean *stop )
return( 0 );
}
/* Create a tile name in the current layout.
/* Make an output object for a tile in the current layout.
*/
static int
tile_name( Layer *layer, char *buf, int x, int y )
static GsfOutput *
tile_name( Layer *layer, int x, int y )
{
/* Need to lock around exists()/mkdir() or there's a race.
*/
static GMutex *file_lock = NULL;
static GOnce file_lock_once = G_ONCE_INIT;
VipsForeignSaveDz *dz = layer->dz;
GsfOutput *out;
char name[VIPS_PATH_MAX];
char dirname[VIPS_PATH_MAX];
char dirname2[VIPS_PATH_MAX];
Layer *p;
int n;
file_lock = g_once( &file_lock_once,
(GThreadFunc) vips_g_mutex_new, NULL );
g_mutex_lock( file_lock );
switch( dz->layout ) {
case VIPS_FOREIGN_DZ_LAYOUT_DZ:
vips_snprintf( dirname, VIPS_PATH_MAX,
"%s_files/%d", dz->basename, layer->n );
vips_snprintf( buf, VIPS_PATH_MAX,
"%s/%d_%d%s", dirname, x, y, dz->suffix );
vips_snprintf( dirname, VIPS_PATH_MAX, "%d", layer->n );
vips_snprintf( name, VIPS_PATH_MAX,
"%d_%d%s", x, y, dz->suffix );
if( !vips_existsf( "%s", dirname ) &&
vips_mkdirf( "%s", dirname ) ) {
g_mutex_unlock( file_lock );
return( -1 );
}
out = vips_gsf_path( dz->tree, name,
dz->root_name, dirname, NULL );
break;
@ -750,54 +922,35 @@ tile_name( Layer *layer, char *buf, int x, int y )
*/
n += y * layer->tiles_across + x;
vips_snprintf( dirname, VIPS_PATH_MAX,
"%s/TileGroup%d", dz->basename, n / 256 );
vips_snprintf( buf, VIPS_PATH_MAX,
"%s/%d-%d-%d%s", dirname, layer->n, x, y, dz->suffix );
vips_snprintf( dirname, VIPS_PATH_MAX, "TileGroup%d", n / 256 );
vips_snprintf( name, VIPS_PATH_MAX,
"%d-%d-%d%s", layer->n, x, y, dz->suffix );
/* Used at the end in ImageProperties.xml
*/
dz->tile_count += 1;
if( !vips_existsf( "%s", dirname ) &&
vips_mkdirf( "%s", dirname ) ) {
g_mutex_unlock( file_lock );
return( -1 );
}
out = vips_gsf_path( dz->tree, name,
dz->root_name, dirname, NULL );
break;
case VIPS_FOREIGN_DZ_LAYOUT_GOOGLE:
vips_snprintf( dirname, VIPS_PATH_MAX,
"%s/%d", dz->basename, layer->n );
vips_snprintf( dirname2, VIPS_PATH_MAX,
"%s/%d", dirname, y );
vips_snprintf( buf, VIPS_PATH_MAX,
"%s/%d%s", dirname2, x, dz->suffix );
vips_snprintf( dirname, VIPS_PATH_MAX, "%d", layer->n );
vips_snprintf( dirname2, VIPS_PATH_MAX, "%d", y );
vips_snprintf( name, VIPS_PATH_MAX, "%d%s", x, dz->suffix );
if( !vips_existsf( "%s", dirname ) &&
vips_mkdirf( "%s", dirname ) ) {
g_mutex_unlock( file_lock );
return( -1 );
}
if( !vips_existsf( "%s", dirname2 ) &&
vips_mkdirf( "%s", dirname2 ) ) {
g_mutex_unlock( file_lock );
return( -1 );
}
out = vips_gsf_path( dz->tree, name,
dz->root_name, dirname, dirname2, NULL );
break;
default:
g_assert( 0 );
g_mutex_unlock( file_lock );
return( -1 );
return( NULL );
}
g_mutex_unlock( file_lock );
return( 0 );
return( out );
}
static int
@ -807,9 +960,11 @@ strip_work( VipsThreadState *state, void *a )
Layer *layer = strip->layer;
VipsForeignSaveDz *dz = layer->dz;
char buf[VIPS_PATH_MAX];
VipsImage *x;
VipsImage *t;
void *buf;
size_t len;
GsfOutput *out;
#ifdef DEBUG_VERBOSE
printf( "strip_work\n" );
@ -837,10 +992,6 @@ strip_work( VipsThreadState *state, void *a )
}
}
if( tile_name( layer, buf,
state->x / dz->tile_size, state->y / dz->tile_size ) )
return( -1 );
#ifdef DEBUG
vips_object_sanity( VIPS_OBJECT( strip->image ) );
#endif /*DEBUG*/
@ -870,11 +1021,34 @@ strip_work( VipsThreadState *state, void *a )
printf( "strip_work: writing to %s\n", buf );
#endif /*DEBUG_VERBOSE*/
if( vips_jpegsave_buffer( x, &buf, &len, NULL ) ) {
g_object_unref( x );
return( -1 );
}
g_object_unref( x );
/* gsf doesn't like more than one write active at once.
*/
g_mutex_lock( vips__global_lock );
out = tile_name( layer,
state->x / dz->tile_size, state->y / dz->tile_size );
gsf_output_write( out, len, buf );
gsf_output_close( out );
g_object_unref( out );
g_mutex_unlock( vips__global_lock );
g_free( buf );
/*
if( vips_image_write_to_file( x, buf ) ) {
g_object_unref( x );
return( -1 );
}
g_object_unref( x );
*/
#ifdef DEBUG_VERBOSE
printf( "strip_work: success\n" );
@ -1291,8 +1465,50 @@ vips_foreign_save_dz_build( VipsObject *object )
if( !(dz->layer = pyramid_build( dz, NULL,
save->ready->Xsize, save->ready->Ysize, &real_pixels )) )
return( -1 );
if( pyramid_mkdir( dz ) )
/* Make the thing we write the tiles into.
*/
{
GsfOutput *out;
GsfOutput *zip;
GError *error = NULL;
if( !(out = gsf_output_stdio_new( dz->basename, &error )) ) {
vips_g_error( &error );
return( -1 );
}
/* This is the zip we are building.
*/
if( !(zip = (GsfOutput *) gsf_outfile_zip_new( out, &error )) ) {
vips_g_error( &error );
return( -1 );
}
/* We can unref @out since @zip has a ref to it.
*/
g_object_unref( out );
dz->tree = vips_gsf_tree_new( zip );
}
/* Drop any path stuff at the start of the output name and remove the
* suffix.
*/
{
char *p;
dz->name = g_path_get_basename( dz->basename );
if( (p = (char *) vips__find_rightmost_brackets( dz->name )) )
*p = '\0';
if( (p = strrchr( dz->name, '.' )) )
*p = '\0';
}
if( dz->layout == VIPS_FOREIGN_DZ_LAYOUT_DZ )
dz->root_name = g_strdup_printf( "%s_files", dz->name );
else
dz->root_name = g_strdup( dz->name );
if( vips_sink_disc( save->ready, pyramid_strip, dz ) )
return( -1 );
@ -1318,6 +1534,9 @@ vips_foreign_save_dz_build( VipsObject *object )
return( -1 );
}
if( vips_gsf_tree_close( dz->tree ) )
return( -1 );
return( 0 );
}

View File

@ -49,6 +49,7 @@ void vips_error_system( int err, const char *domain, const char *fmt, ... )
void vips_verror_system( int err, const char *domain,
const char *fmt, va_list ap );
void vips_error_g( GError **error );
void vips_g_error( GError **error );
void vips_warn( const char *domain, const char *fmt, ... )
__attribute__((format(printf, 2, 3)));
void vips_vwarn( const char *domain, const char *fmt, va_list ap );

View File

@ -311,7 +311,9 @@ vips_error_system( int err, const char *domain, const char *fmt, ... )
* This function sets the glib error pointer from the vips error buffer and
* clears it. It's handy for returning errors to glib functions from vips.
*
* See also: g_set_error().
* See vips_g_error() for the inverse operation.
*
* See also: g_set_error(), vips_g_error().
*/
void
vips_error_g( GError **error )
@ -331,6 +333,27 @@ vips_error_g( GError **error )
vips_error_clear();
}
/**
* vips_g_error:
* @error: glib error pointer
*
* This function adds the %GError to the vips error buffer and clears it. It's
* the opposite of vips_error_g().
*
* See also: vips_error_g().
*/
void
vips_g_error( GError **error )
{
if( error &&
*error ) {
vips_error( "glib", "%s\n", (*error)->message );
g_error_free( *error );
*error = NULL;
}
}
/**
* vips_error_clear:
*

View File

@ -76,6 +76,8 @@
#include <limits.h>
#include <string.h>
#include <gsf/gsf.h>
#include <vips/vips.h>
#include <vips/thread.h>
#include <vips/internal.h>
@ -304,6 +306,10 @@ vips__init( const char *argv0 )
*/
vips_vector_init();
/* Use this for structured file write.
*/
gsf_init();
/* Register vips_shutdown(). This may well not get called and many
* platforms don't support it anyway.
*/
@ -429,6 +435,8 @@ vips_shutdown( void )
vips__thread_profile_stop();
gsf_shutdown();
/* In dev releases, always show leaks. But not more than once, it's
* annoying.
*/

View File

@ -200,10 +200,8 @@ vips_g_thread_new( const char *domain, GThreadFunc func, gpointer data )
#endif
if( !thread ) {
if( error ) {
vips_error( domain, "%s", error->message );
g_error_free( error );
}
if( error )
vips_g_error( &error );
else
vips_error( domain,
"%s", _( "unable to create thread" ) );