add dbuf object

dynamically expanding buffer

also, escape "<>& appropriately when we write xml
This commit is contained in:
John Cupitt 2017-02-26 17:37:46 +00:00
parent 5614330cc4
commit e1b9c789cb
8 changed files with 388 additions and 238 deletions

View File

@ -36,7 +36,6 @@
more tightly ... this makes seq more versatile and more reliable
- use expat, not libxml, for XML load ... removes a required dependency, since
we get expat as part of glib
- use printf for xml write
8/12/16 started 8.4.5
- allow libgsf-1.14.26 to help centos, thanks tdiprima

134
TODO
View File

@ -1,123 +1,7 @@
- remove libxml from configure.ac
we don't escape " inside xml attr strings, or < > inside xml content
- use VipsDbuf for tiffsave_buffer etc.
break Buffer out into a separate class that does this? VipsDbuf, for dynamic
buffer? or can it be a mode for VipsBuf?
- verify xml data against master for vips save and dzsave
buffer should expand by 30% on each fill to avoid lots of tiny realloc
use for tiffsave_buffer etc.
have things like write-escaped-string as extra append operations
need one that escapes ", a separate one that escapes < >
- dzsave vips.properties is currently
<?xml version="1.0"?>
<image xmlns="http://www.vips.ecs.soton.ac.uk/dzsave" date="2017-02-25T17:32:07.565137Z" version="8.5.0">
<properties>
<property>
<name>width</name>
<value type="gint">1450</value>
</property>
<property>
<name>height</name>
<value type="gint">2048</value>
</property>
<property>
<name>bands</name>
<value type="gint">3</value>
</property>
<property>
<name>xoffset</name>
<value type="gint">0</value>
</property>
<property>
<name>yoffset</name>
<value type="gint">0</value>
</property>
<property>
<name>xres</name>
<value type="gdouble">0.99999999990686772</value>
</property>
<property>
<name>yres</name>
<value type="gdouble">0.99999999990686772</value>
</property>
<property>
<name>vips-loader</name>
<value type="VipsRefString">jpegload</value>
</property>
<property>
<name>jpeg-multiscan</name>
<value type="gint">0</value>
</property>
<property>
<name>exif-data</name>
<value type="VipsBlob">
RXhpZgAASUkqAAgAAAAGABIBAwABAAAAAQAAABoBBQABAAAAVgAAABsBBQABAAAAXgAAACgBAwAB
AAAAAgAAABMCAwABAAAAAQAAAGmHBAABAAAAZgAAAAAAAAAR/P//KSgUChH8//8pKBQKBgAAkAcA
BAAAADAyMTABkQcABAAAAAECAwAAoAcABAAAADAxMDABoAMAAQAAAP//AAACoAQAAQAAAKoFAAAD
oAQAAQAAAAAIAAAAAAAA
</value>
</property>
<property>
<name>resolution-unit</name>
<value type="VipsRefString">in</value>
</property>
<property>
<name>exif-ifd0-Orientation</name>
<value type="VipsRefString">1 (Top-left, Short, 1 components, 2 bytes)</value>
</property>
<property>
<name>exif-ifd0-XResolution</name>
<value type="VipsRefString">4294966289/169093161 (25.399999998, Rational, 1 components, 8 bytes)</value>
</property>
<property>
<name>exif-ifd0-YResolution</name>
<value type="VipsRefString">4294966289/169093161 (25.399999998, Rational, 1 components, 8 bytes)</value>
</property>
<property>
<name>exif-ifd0-ResolutionUnit</name>
<value type="VipsRefString">2 (Inch, Short, 1 components, 2 bytes)</value>
</property>
<property>
<name>exif-ifd0-YCbCrPositioning</name>
<value type="VipsRefString">1 (Centred, Short, 1 components, 2 bytes)</value>
</property>
<property>
<name>exif-ifd2-ExifVersion</name>
<value type="VipsRefString">Exif Version 2.1 (Exif Version 2.1, Undefined, 4 components, 4 bytes)</value>
</property>
<property>
<name>exif-ifd2-ComponentsConfiguration</name>
<value type="VipsRefString">Y Cb Cr - (Y Cb Cr -, Undefined, 4 components, 4 bytes)</value>
</property>
<property>
<name>exif-ifd2-FlashPixVersion</name>
<value type="VipsRefString">FlashPix Version 1.0 (FlashPix Version 1.0, Undefined, 4 components, 4 bytes)</value>
</property>
<property>
<name>exif-ifd2-ColorSpace</name>
<value type="VipsRefString">65535 (Internal error (unknown value 65535), Short, 1 components, 2 bytes)</value>
</property>
<property>
<name>exif-ifd2-PixelXDimension</name>
<value type="VipsRefString">1450 (1450, Long, 1 components, 4 bytes)</value>
</property>
<property>
<name>exif-ifd2-PixelYDimension</name>
<value type="VipsRefString">2048 (2048, Long, 1 components, 4 bytes)</value>
</property>
<property>
<name>orientation</name>
<value type="gint">1</value>
</property>
</properties>
</image>
@ -246,20 +130,6 @@ DEBUG:gi.overrides.Vips:assigning sequential
output imagevec
output blob
- fix up aconv
- rewrite im_conv() etc. as vips_conv(), also the mosaicing functions
finally finish --disable-deprecated option
- can we use postbuild elsewhere? look at use of "preclose" / "written", etc.
- test draw_mask on labq images
we probably need to unpack the ink back to double before blending

View File

@ -6,6 +6,7 @@ pkginclude_HEADERS = \
deprecated.h \
arithmetic.h \
buf.h \
dbuf.h \
colour.h \
conversion.h \
convolution.h \

View File

@ -0,0 +1,72 @@
/* A dynamic memory buffer that expands as you write.
*/
/*
This file is part of VIPS.
VIPS is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301 USA
*/
/*
These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk
*/
#ifndef VIPS_DBUF_H
#define VIPS_DBUF_H
#ifdef __cplusplus
extern "C" {
#endif /*__cplusplus*/
#include <vips/vips.h>
/* A buffer in the process of being written to ... multiple calls to
* vips_dbuf_append add to it.
*/
typedef struct _VipsDbuf {
/* All fields are private.
*/
/*< private >*/
/* The current base, and the size of the allocated memory area.
*/
char *data;
size_t max_size;
/* And the write point.
*/
size_t write_point;
} VipsDbuf;
void vips_dbuf_destroy( VipsDbuf *buf );
void vips_dbuf_init( VipsDbuf *buf );
gboolean vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size );
gboolean vips_dbuf_appendf( VipsDbuf *dbuf, const char *fmt, ... );
void vips_dbuf_rewind( VipsDbuf *dbuf );
void vips_dbuf_destroy( VipsDbuf *dbuf );
char *vips_dbuf_string( VipsDbuf *dbuf, size_t *size );
char *vips_dbuf_steal( VipsDbuf *dbuf, size_t *size );
#endif /*VIPS_DBUF_H*/
#ifdef __cplusplus
}
#endif /*__cplusplus*/

View File

@ -106,6 +106,7 @@ extern "C" {
#include <vips/basic.h>
#include <vips/buf.h>
#include <vips/dbuf.h>
#include <vips/util.h>
#include <vips/object.h>
#include <vips/type.h>

View File

@ -1,6 +1,7 @@
noinst_LTLIBRARIES = libiofuncs.la
libiofuncs_la_SOURCES = \
dbuf.c \
reorder.c \
vipsmarshal.h \
vipsmarshal.c \

206
libvips/iofuncs/dbuf.c Normal file
View File

@ -0,0 +1,206 @@
/* A dynamic memory buffer that expands as you write.
*/
/*
Copyright (C) 1991-2003 The National Gallery
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301 USA
*/
/*
These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk
*/
/*
#define DEBUG
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif /*HAVE_CONFIG_H*/
#include <vips/intl.h>
#include <string.h>
#include <vips/vips.h>
/**
* vips_dbuf_init:
* @dbuf: the buffer
*
* Initialize a buffer.
*/
void
vips_dbuf_init( VipsDbuf *dbuf )
{
dbuf->data = NULL;
dbuf->max_size = 0;
dbuf->write_point = 0;
}
/**
* vips_dbuf_append:
* @dbuf: the buffer
* @data: the data to append to the buffer
* @size: the size of the len to append
*
* Append len bytes from @data to the buffer. The buffer expands if necessary.
*
* Returns: %FALSE on out of memory, %TRUE otherwise.
*/
gboolean
vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size )
{
size_t new_write_point = dbuf->write_point + size;
if( new_write_point > dbuf->max_size ) {
size_t new_max_size = 3 * (16 + new_write_point) / 2;
char *new_data;
if( !(new_data = g_try_realloc( dbuf->data, new_max_size )) ) {
vips_error( "VipsDbuf", "%s", _( "out of memory" ) );
return( FALSE );
}
dbuf->data = new_data;
dbuf->max_size = new_max_size;
}
memcpy( dbuf->data + dbuf->write_point, data, size );
dbuf->write_point = new_write_point;
return( TRUE );
}
/**
* vips_dbuf_appendf:
* @dbuf: the buffer
* @fmt: <function>printf()</function>-style format string
* @...: arguments to format string
*
* Format the string and append to @dbuf.
*
* Returns: %FALSE on out of memory, %TRUE otherwise.
*/
gboolean
vips_dbuf_appendf( VipsDbuf *dbuf, const char *fmt, ... )
{
va_list ap;
char *line;
va_start( ap, fmt );
line = g_strdup_vprintf( fmt, ap );
va_end( ap );
if( vips_dbuf_append( dbuf, line, strlen( line ) ) ) {
g_free( line );
return( FALSE );
}
g_free( line );
return( TRUE );
}
/**
* vips_dbuf_rewind:
* @dbuf: the buffer
*
* Reset the buffer to empty. No memory is freed, just the write pointer is
* reset.
*/
void
vips_dbuf_rewind( VipsDbuf *dbuf )
{
dbuf->write_point = 0;
}
/**
* vips_dbuf_destroy:
* @dbuf: the buffer
*
* Destroy a buffer. This frees any allocated memory.
*/
void
vips_dbuf_destroy( VipsDbuf *dbuf )
{
VIPS_FREE( dbuf->data );
dbuf->max_size = 0;
dbuf->write_point = 0;
}
/**
* vips_dbuf_steal:
* @dbuf: the buffer
* @size: (allow-none): optionally return length in bytes here
*
* Destroy a buffer, but rather than freeing memory, a pointer is returned.
* This must be freed with g_free().
*
* A `\0` is appended, but not included in the character count. This is so the
* pointer can be safely treated as a C string.
*
* Returns: (transfer full): The pointer held by @dbuf.
*/
char *
vips_dbuf_steal( VipsDbuf *dbuf, size_t *size )
{
char *data;
vips_dbuf_append( dbuf, "", 1 );
dbuf->write_point -= 1;
data = dbuf->data;
if( size )
*size = dbuf->write_point;
dbuf->data = NULL;
dbuf->max_size = 0;
dbuf->write_point = 0;
return( data );
}
/**
* vips_dbuf_data:
* @dbuf: the buffer
* @size: (allow-none): optionally return length in bytes here
*
* Return a pointer to @dbuf's internal data.
*
* A `\0` is appended, but not included in the character count. This is so the
* pointer can be safely treated as a C string.
*
* Returns: (transfer none): The pointer held by @dbuf.
*/
char *
vips_dbuf_string( VipsDbuf *dbuf, size_t *size )
{
vips_dbuf_append( dbuf, "", 1 );
dbuf->write_point -= 1;
if( size )
*size = dbuf->write_point;
return( dbuf->data );
}

View File

@ -49,8 +49,8 @@
*/
/*
#define DEBUG
#define SHOW_HEADER
#define DEBUG
*/
#ifdef HAVE_CONFIG_H
@ -492,56 +492,6 @@ parser_read_fd( XML_Parser parser, int fd )
#define MAX_PARSE_ATTR (256)
/* A memory buffer that expands as we write to it.
*/
typedef struct _Buffer {
char *data;
size_t current_size;
} Buffer;
static void
buffer_init( Buffer *buffer )
{
buffer->data = NULL;
buffer->current_size = 0;
}
static void
buffer_append( Buffer *buffer, const char *data, int len )
{
size_t new_size = buffer->current_size + len;
buffer->data = g_realloc( buffer->data, new_size );
memcpy( buffer->data + buffer->current_size, data, len );
buffer->current_size = new_size;
}
static void
buffer_appendf( Buffer *buffer, const char *fmt, ... )
{
va_list ap;
char line[256];
va_start( ap, fmt );
(void) vips_vsnprintf( line, 256, fmt, ap );
va_end( ap );
buffer_append( buffer, line, strlen( line ) );
}
static void
buffer_rewind( Buffer *buffer )
{
buffer->current_size = 0;
}
static void
buffer_destroy( Buffer *buffer )
{
VIPS_FREE( buffer->data );
buffer->current_size = 0;
}
/* What we track during expat parse.
*/
typedef struct _VipsExpatParse {
@ -562,7 +512,7 @@ typedef struct _VipsExpatParse {
/* Accumulate data here.
*/
Buffer buffer;
VipsDbuf dbuf;
} VipsExpatParse;
static void
@ -586,7 +536,7 @@ parser_element_start_handler( void *user_data,
vips_strncpy( vep->type, p[1], MAX_PARSE_ATTR );
}
buffer_rewind( &vep->buffer );
vips_dbuf_rewind( &vep->dbuf );
}
else if( strcmp( name, "header" ) == 0 )
vep->header = TRUE;
@ -666,16 +616,10 @@ parser_element_end_handler( void *user_data, const XML_Char *name )
#endif /*DEBUG*/
if( strcmp( name, "field" ) == 0 ) {
buffer_append( &vep->buffer, "", 1 );
#ifdef DEBUG
printf( "parser_element_end_handler: %zd bytes\n",
vep->current_size );
#endif /*DEBUG*/
if( vep->header ) {
if( strcmp( name, "Hist" ) == 0 )
set_history( vep->image, vep->buffer.data );
set_history( vep->image,
vips_dbuf_string( &vep->dbuf, NULL ) );
}
else {
GType gtype = g_type_from_name( vep->type );
@ -686,7 +630,8 @@ parser_element_end_handler( void *user_data, const XML_Char *name )
g_value_type_transformable(
VIPS_TYPE_SAVE_STRING, gtype ) &&
set_meta( vep->image,
gtype, vep->name, vep->buffer.data ) )
gtype, vep->name,
vips_dbuf_string( &vep->dbuf, NULL ) ) )
vep->error = TRUE;
}
}
@ -701,7 +646,7 @@ parser_data_handler( void *user_data, const XML_Char *data, int len )
printf( "parser_data_handler: %d bytes\n", len );
#endif /*DEBUG*/
buffer_append( &vep->buffer, data, len );
vips_dbuf_append( &vep->dbuf, data, len );
}
/* Called at the end of vips open ... get any XML after the pixel data
@ -719,7 +664,7 @@ readhist( VipsImage *im )
parser = XML_ParserCreate( "UTF-8" );
vep.image = im;
buffer_init( &vep.buffer );
vips_dbuf_init( &vep.dbuf );
vep.error = FALSE;
XML_SetUserData( parser, &vep );
@ -729,12 +674,12 @@ readhist( VipsImage *im )
if( parser_read_fd( parser, im->fd ) ||
vep.error ) {
buffer_destroy( &vep.buffer );
vips_dbuf_destroy( &vep.dbuf );
XML_ParserFree( parser );
return( -1 );
}
buffer_destroy( &vep.buffer );
vips_dbuf_destroy( &vep.dbuf );
XML_ParserFree( parser );
return( 0 );
@ -768,8 +713,59 @@ vips__write_extension_block( VipsImage *im, void *buf, int size )
return( 0 );
}
/* Append a string to a buffer, but escape " as \".
*/
static void
dbuf_append_quotes( VipsDbuf *dbuf, const char *str )
{
const char *p;
size_t len;
for( p = str; *p; p += len ) {
len = strcspn( p, "\"" );
vips_dbuf_append( dbuf, p, len );
if( p[len] == '"' )
vips_dbuf_append( dbuf, "\\", 1 );
}
}
/* Append a string to a buffer, but escape &<>.
*/
static void
dbuf_append_amp( VipsDbuf *dbuf, const char *str )
{
const char *p;
size_t len;
for( p = str; *p; p += len ) {
len = strcspn( p, "&<>" );
vips_dbuf_append( dbuf, p, len );
switch( p[len] ) {
case '&':
vips_dbuf_append( dbuf, "&amp;", 5 );
len += 1;
break;
case '<':
vips_dbuf_append( dbuf, "&lt;", 4 );
len += 1;
break;
case '>':
vips_dbuf_append( dbuf, "&gt;", 4 );
len += 1;
break;
default:
break;
}
}
}
static void *
build_xml_meta( VipsMeta *meta, Buffer *buffer )
build_xml_meta( VipsMeta *meta, VipsDbuf *dbuf )
{
GType type = G_VALUE_TYPE( &meta->value );
@ -786,14 +782,16 @@ build_xml_meta( VipsMeta *meta, Buffer *buffer )
if( !g_value_transform( &meta->value, &save_value ) ) {
vips_error( "VipsImage", "%s",
_( "error transforming to save format" ) );
return( buffer );
return( meta );
}
str = vips_value_get_save_string( &save_value );
buffer_appendf( buffer, " <field type=\"%s\" name=\"%s\">",
g_type_name( type ), meta->name );
buffer_append( buffer, str, strlen( str ) );
buffer_appendf( buffer, "</field>\n" );
vips_dbuf_appendf( dbuf, " <field type=\"%s\" name=\"",
g_type_name( type ) );
dbuf_append_quotes( dbuf, meta->name );
vips_dbuf_appendf( dbuf, "\">" );
dbuf_append_amp( dbuf, str );
vips_dbuf_appendf( dbuf, "</field>\n" );
g_value_unset( &save_value );
}
@ -806,43 +804,43 @@ build_xml_meta( VipsMeta *meta, Buffer *buffer )
static char *
build_xml( VipsImage *image )
{
Buffer buffer;
VipsDbuf dbuf;
const char *str;
buffer_init( &buffer );
vips_dbuf_init( &dbuf );
buffer_appendf( &buffer, "<?xml version=\"1.0\"?>\n" );
buffer_appendf( &buffer, "<root xmlns=\"%s/vips/%d.%d.%d\">\n",
vips_dbuf_appendf( &dbuf, "<?xml version=\"1.0\"?>\n" );
vips_dbuf_appendf( &dbuf, "<root xmlns=\"%s/vips/%d.%d.%d\">\n",
NAMESPACE_URI,
VIPS_MAJOR_VERSION, VIPS_MINOR_VERSION, VIPS_MICRO_VERSION );
buffer_appendf( &buffer, " <header>\n" );
vips_dbuf_appendf( &dbuf, " <header>\n" );
str = vips_image_get_history( image );
buffer_appendf( &buffer, " <field type=\"%s\" name=\"Hist\">",
vips_dbuf_appendf( &dbuf, " <field type=\"%s\" name=\"Hist\">",
g_type_name( VIPS_TYPE_REF_STRING ) );
buffer_append( &buffer, str, strlen( str ) );
buffer_appendf( &buffer, "</field>\n" );
dbuf_append_amp( &dbuf, str );
vips_dbuf_appendf( &dbuf, "</field>\n" );
buffer_appendf( &buffer, " </header>\n" );
buffer_appendf( &buffer, " <meta>\n" );
vips_dbuf_appendf( &dbuf, " </header>\n" );
vips_dbuf_appendf( &dbuf, " <meta>\n" );
if( vips_slist_map2( image->meta_traverse,
(VipsSListMap2Fn) build_xml_meta, &buffer, NULL ) ) {
buffer_destroy( &buffer );
(VipsSListMap2Fn) build_xml_meta, &dbuf, NULL ) ) {
vips_dbuf_destroy( &dbuf );
return( NULL );
}
buffer_appendf( &buffer, " </meta>\n" );
buffer_appendf( &buffer, "</root>\n" );
vips_dbuf_appendf( &dbuf, " </meta>\n" );
vips_dbuf_appendf( &dbuf, "</root>\n" );
return( buffer.data );
return( vips_dbuf_steal( &dbuf, NULL ) );
}
static void *
vips__xml_properties_meta( VipsImage *image,
const char *field, GValue *value, void *a )
{
Buffer *buffer = (Buffer *) a;
VipsDbuf *dbuf = (VipsDbuf *) a;
GType type = G_VALUE_TYPE( value );
const char *str;
@ -858,18 +856,20 @@ vips__xml_properties_meta( VipsImage *image,
if( !g_value_transform( value, &save_value ) ) {
vips_error( "VipsImage", "%s",
_( "error transforming to save format" ) );
return( buffer );
return( dbuf );
}
str = vips_value_get_save_string( &save_value );
g_value_unset( &save_value );
buffer_appendf( buffer, " <property>\n" );
buffer_appendf( buffer, " <name>%s</name>\n", field );
buffer_appendf( buffer, " <value type=\"%s\">",
vips_dbuf_appendf( dbuf, " <property>\n" );
vips_dbuf_appendf( dbuf, " <name>" );
dbuf_append_amp( dbuf, field );
vips_dbuf_appendf( dbuf, "</name>\n" );
vips_dbuf_appendf( dbuf, " <value type=\"%s\">",
g_type_name( type ) );
buffer_append( buffer, str, strlen( str ) );
buffer_appendf( buffer, "</value>\n" );
buffer_appendf( buffer, " </property>\n" );
dbuf_append_amp( dbuf, str );
vips_dbuf_appendf( dbuf, "</value>\n" );
vips_dbuf_appendf( dbuf, " </property>\n" );
}
return( NULL );
@ -881,32 +881,32 @@ vips__xml_properties_meta( VipsImage *image,
char *
vips__xml_properties( VipsImage *image )
{
Buffer buffer;
VipsDbuf dbuf;
GTimeVal now;
char *date;
buffer_init( &buffer );
vips_dbuf_init( &dbuf );
g_get_current_time( &now );
date = g_time_val_to_iso8601( &now );
buffer_appendf( &buffer, "<?xml version=\"1.0\"?>\n" );
buffer_appendf( &buffer, "<image xmlns=\"%s/dzsave\" "
vips_dbuf_appendf( &dbuf, "<?xml version=\"1.0\"?>\n" );
vips_dbuf_appendf( &dbuf, "<image xmlns=\"%s/dzsave\" "
"date=\"%s\" version=\"%d.%d.%d\">\n",
NAMESPACE_URI,
date,
VIPS_MAJOR_VERSION, VIPS_MINOR_VERSION, VIPS_MICRO_VERSION );
g_free( date );
buffer_appendf( &buffer, " <properties>\n" );
vips_dbuf_appendf( &dbuf, " <properties>\n" );
if( vips_image_map( image, vips__xml_properties_meta, &buffer ) ) {
buffer_destroy( &buffer );
if( vips_image_map( image, vips__xml_properties_meta, &dbuf ) ) {
vips_dbuf_destroy( &dbuf );
return( NULL );
}
buffer_appendf( &buffer, " </properties>\n" );
buffer_appendf( &buffer, "</image>\n" );
vips_dbuf_appendf( &dbuf, " </properties>\n" );
vips_dbuf_appendf( &dbuf, "</image>\n" );
return( buffer.data );
return( vips_dbuf_steal( &dbuf, NULL ) );
}
/* Append XML to output fd.