sergiotarxz/libvips
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

started png and jpg with dbuf

Browse Source 
rad and webp still to do, maybe tiff
This commit is contained in:
John Cupitt 2017-02-27 22:06:22 +00:00
parent e1b9c789cb
commit 40294bb85c
7 changed files with 137 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -46,8 +46,7 @@ Untar, then in the libvips directory you should just be able to do:
$ ./configure
Check the summary at the end of `configure` carefully.
libvips must have `build-essential`, `pkg-config`, `glib2.0-dev`, and
`libxml2-dev`.
libvips must have `build-essential`, `pkg-config`, `glib2.0-dev`.
For the vips8 Python binding, you must have
`gobject-introspection`, `python-gi-dev`, and `libgirepository1.0-dev`.
@ -145,8 +144,7 @@ Static analysis with:
# Dependencies
libvips has to have `gettext`, `glib2.0-dev` and `libxml2-dev`. Other
dependencies are optional, see below.
libvips has to have `glib2.0-dev`. Other dependencies are optional, see below.
# Optional dependencies

20
TODO
View File

@ -1,5 +1,25 @@
- use VipsDbuf for tiffsave_buffer etc.
$ grep -l save_buffer *.c
dzsave.c
jpegsave.c
pngsave.c
radsave.c
tiffsave.c
webpsave.c
done png, jpg
tiff needs seek ... perhaps add this to dbuf?
dzsave uses gsf
rad and webp left to do
vips2jpeg term_destination() needs some kind of truncate call for dbuf
- verify xml data against master for vips save and dzsave

200
libvips/foreign/vips2jpeg.c
View File

@ -82,6 +82,8 @@
* - turn off chroma subsample for Q > 90
* 7/11/16
* - move exif handling out to exif.c
* 27/2/17
* - use dbuf for memory output
*/
/*
@ -670,112 +672,6 @@ vips__jpeg_write_file( VipsImage *in,
return( 0 );
}
/* We can't predict how large the output buffer we need is, because we might
* need space for ICC profiles and stuff. So we write to a linked list of mem
* buffers and add a new one as they fill.
*/
#define BUFFER_SIZE (10000)
/* A buffer.
*/
typedef struct _Block {
j_compress_ptr cinfo;
struct _Block *first;
struct _Block *next;
JOCTET *data; /* Allocated area */
size_t size; /* Max size */
size_t used; /* How much has been used */
} Block;
static Block *
block_new( j_compress_ptr cinfo )
{
Block *block;
block = (Block *) (*cinfo->mem->alloc_large)
( (j_common_ptr) cinfo, JPOOL_IMAGE, sizeof( Block ) );
block->cinfo = cinfo;
block->first = block;
block->next = NULL;
block->data = (JOCTET *) (*cinfo->mem->alloc_large)
( (j_common_ptr) cinfo, JPOOL_IMAGE, BUFFER_SIZE );
block->size = BUFFER_SIZE;
block->used = 0;
return( block );
}
static Block *
block_last( Block *block )
{
while( block->next )
block = block->next;
return( block );
}
static Block *
block_append( Block *block )
{
Block *new;
g_assert( block );
new = block_new( block->cinfo );
new->first = block->first;
block_last( block )->next = new;
return( new );
}
static size_t
block_length( Block *block )
{
size_t len;
len = 0;
for( block = block->first; block; block = block->next )
len += block->used;
return( len );
}
static void
block_copy( Block *block, void *dest )
{
JOCTET *p;
p = dest;
for( block = block->first; block; block = block->next ) {
memcpy( p, block->data, block->used );
p += block->used;
}
}
#ifdef DEBUG
static void
block_print( Block *block )
{
int i;
printf( "total length = %zd\n", block_length( block ) );
printf( "set of blocks:\n" );
i = 0;
for( block = block->first; block; block = block->next ) {
printf( "%d) %p, first = %p, next = %p"
"\t data = %p, size = %zd, used = %zd\n",
i, block, block->first, block->next,
block->data, block->size, block->used );
i += 1;
}
}
#endif /*DEBUG*/
/* Just like the above, but we write to a memory buffer.
*
* A memory buffer for the compressed image.
@ -788,16 +684,36 @@ typedef struct {
/* Private stuff during write.
*/
/* Build the output area here in chunks.
/* Build the output area here.
*/
Block *block;
VipsDbuf dbuf;
/* Copy the compressed area here.
/* Write the generated area here.
*/
void **obuf; /* Allocated buffer, and size */
size_t *olen;
} OutputBuffer;
/* Buffer full method ... allocate a new output block. This is only called
* when the output area is exactly full.
*/
METHODDEF(boolean)
empty_output_buffer( j_compress_ptr cinfo )
{
OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
size_t size;
vips_dbuf_allocate( &buf->dbuf, 10000 );
buf->pub.next_output_byte =
(JOCTET *) vips_dbuf_get_write( &buf->dbuf, &size );
buf->pub.free_in_buffer = size;
/* TRUE means we've made some more space.
*/
return( 1 );
}
/* Init dest method.
*/
METHODDEF(void)
@ -805,38 +721,8 @@ init_destination( j_compress_ptr cinfo )
{
OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
/* Allocate relative to the image we are writing .. freed when we junk
* this output.
*/
buf->block = block_new( cinfo );
/* Set buf pointers for library.
*/
buf->pub.next_output_byte = buf->block->data;
buf->pub.free_in_buffer = buf->block->size;
}
/* Buffer full method ... allocate a new output block.
*/
METHODDEF(boolean)
empty_output_buffer( j_compress_ptr cinfo )
{
OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
/* Record how many bytes we used. empty_output_buffer() is always
* called when the buffer is exactly full.
*/
buf->block->used = buf->block->size;
/* New block and reset.
*/
buf->block = block_append( buf->block );
buf->pub.next_output_byte = buf->block->data;
buf->pub.free_in_buffer = buf->block->size;
/* TRUE means we've made some more space.
*/
return( 1 );
vips_dbuf_init( &buf->dbuf );
empty_output_buffer( cinfo );
}
/* Cleanup. Copy the set of blocks out as a big lump.
@ -846,35 +732,15 @@ term_destination( j_compress_ptr cinfo )
{
OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
size_t len;
void *obuf;
size_t size;
/* Record the number of bytes we wrote in the final buffer.
* pub.free_in_buffer is valid here.
/* This can be called before the output area fills. We need to chop
* size down to the number of bytes actually written.
*/
buf->block->used = buf->block->size - buf->pub.free_in_buffer;
buf->dbuf.write_point = buf->dbuf.max_size - buf->pub.free_in_buffer;
#ifdef DEBUG
block_print( buf->block );
#endif /*DEBUG*/
/* ... and we can count up our buffers now.
*/
len = block_length( buf->block );
/* Allocate and copy to the output area.
*/
if( !(obuf = vips_malloc( NULL, len )) )
ERREXIT( cinfo, JERR_FILE_WRITE );
else {
/* coverity doesn't know ERREXIT() does not return, so put
* this in an else.
*/
*(buf->obuf) = obuf;
*(buf->olen) = len;
block_copy( buf->block, obuf );
}
*(buf->obuf) = vips_dbuf_steal( &buf->dbuf, &size );
*(buf->olen) = size;
}
/* Set dest to one of our objects.

49
libvips/foreign/vipspng.c
View File

@ -59,6 +59,8 @@
* - support --strip option
* 17/1/17
* - invalidate operation on read error
* 27/2/17
* - use dbuf for buffer output
*/
/*
@ -734,10 +736,7 @@ typedef struct {
VipsImage *memory;
FILE *fp;
char *buf;
size_t len;
size_t alloc;
VipsDbuf dbuf;
png_structp pPng;
png_infop pInfo;
@ -749,7 +748,7 @@ write_finish( Write *write )
{
VIPS_FREEF( fclose, write->fp );
VIPS_UNREF( write->memory );
VIPS_FREE( write->buf );
vips_dbuf_destroy( &write->dbuf );
if( write->pPng )
png_destroy_write_struct( &write->pPng, &write->pInfo );
}
@ -771,9 +770,7 @@ write_new( VipsImage *in )
write->in = in;
write->memory = NULL;
write->fp = NULL;
write->buf = NULL;
write->len = 0;
write->alloc = 0;
vips_dbuf_init( &write->dbuf );
g_signal_connect( in, "close",
G_CALLBACK( write_destroy ), write );
@ -1014,41 +1011,12 @@ vips__png_write( VipsImage *in, const char *filename,
return( 0 );
}
static void
write_grow( Write *write, size_t grow_len )
{
size_t new_len = write->len + grow_len;
if( new_len > write->alloc ) {
size_t proposed_alloc = (16 + write->alloc) * 3 / 2;
write->alloc = VIPS_MAX( proposed_alloc, new_len );
/* Our result must be freedd with g_free(), so it's OK to use
* g_realloc().
*/
write->buf = g_realloc( write->buf, write->alloc );
VIPS_DEBUG_MSG( "write_buf_grow: grown to %zd bytes\n",
write->alloc );
}
}
static void
user_write_data( png_structp png_ptr, png_bytep data, png_size_t length )
{
Write *write = (Write *) png_get_io_ptr( png_ptr );
char *write_start;
write_grow( write, length );
write_start = write->buf + write->len;
memcpy( write_start, data, length );
write->len += length;
g_assert( write->len <= write->alloc );
vips_dbuf_append( &write->dbuf, data, length );
}
int
@ -1073,10 +1041,7 @@ vips__png_write_buf( VipsImage *in,
return( -1 );
}
*obuf = write->buf;
write->buf = NULL;
if( olen )
*olen = write->len;
*obuf = vips_dbuf_steal( &write->dbuf, olen );
write_finish( write );

11
libvips/include/vips/dbuf.h
View File

@ -48,7 +48,7 @@ typedef struct _VipsDbuf {
/* The current base, and the size of the allocated memory area.
*/
char *data;
unsigned char *data;
size_t max_size;
/* And the write point.
@ -58,12 +58,15 @@ typedef struct _VipsDbuf {
void vips_dbuf_destroy( VipsDbuf *buf );
void vips_dbuf_init( VipsDbuf *buf );
gboolean vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size );
gboolean vips_dbuf_allocate( VipsDbuf *dbuf, size_t size );
unsigned char *vips_dbuf_get_write( VipsDbuf *dbuf, size_t *size );
gboolean vips_dbuf_append( VipsDbuf *dbuf,
const unsigned char *data, size_t size );
gboolean vips_dbuf_appendf( VipsDbuf *dbuf, const char *fmt, ... );
void vips_dbuf_rewind( VipsDbuf *dbuf );
void vips_dbuf_destroy( VipsDbuf *dbuf );
char *vips_dbuf_string( VipsDbuf *dbuf, size_t *size );
char *vips_dbuf_steal( VipsDbuf *dbuf, size_t *size );
unsigned char *vips_dbuf_string( VipsDbuf *dbuf, size_t *size );
unsigned char *vips_dbuf_steal( VipsDbuf *dbuf, size_t *size );
#endif /*VIPS_DBUF_H*/

69
libvips/iofuncs/dbuf.c
View File

@ -56,24 +56,23 @@ vips_dbuf_init( VipsDbuf *dbuf )
}
/**
* vips_dbuf_append:
* vips_dbuf_allocate:
* @dbuf: the buffer
* @data: the data to append to the buffer
* @size: the size of the len to append
* @size: the size to allocate
*
* Append len bytes from @data to the buffer. The buffer expands if necessary.
* Make sure @dbuf has at least @size bytes available for writing.
*
* Returns: %FALSE on out of memory, %TRUE otherwise.
*/
gboolean
vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size )
vips_dbuf_allocate( VipsDbuf *dbuf, size_t size )
{
size_t new_write_point = dbuf->write_point + size;
if( new_write_point > dbuf->max_size ) {
size_t new_max_size = 3 * (16 + new_write_point) / 2;
char *new_data;
unsigned char *new_data;
if( !(new_data = g_try_realloc( dbuf->data, new_max_size )) ) {
vips_error( "VipsDbuf", "%s", _( "out of memory" ) );
@ -84,8 +83,50 @@ vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size )
dbuf->max_size = new_max_size;
}
return( TRUE );
}
/**
* vips_dbuf_get_write:
* @dbuf: the buffer
* @size: (allow-none): optionally return length in bytes here
*
* Return a pointer to an area you can write to, return length of area in
* @size. Use vips_dbuf_allocate() before this call to make the space.
*
* Returns: (transfer none): start of write area.
*/
unsigned char *
vips_dbuf_get_write( VipsDbuf *dbuf, size_t *size )
{
unsigned char *data = dbuf->data + dbuf->write_point;
if( size )
*size = dbuf->max_size - dbuf->write_point;
dbuf->write_point = dbuf->max_size;
return( data );
}
/**
* vips_dbuf_append:
* @dbuf: the buffer
* @data: the data to append to the buffer
* @size: the size of the len to append
*
* Append len bytes from @data to the buffer. The buffer expands if necessary.
*
* Returns: %FALSE on out of memory, %TRUE otherwise.
*/
gboolean
vips_dbuf_append( VipsDbuf *dbuf, const unsigned char *data, size_t size )
{
if( !vips_dbuf_allocate( dbuf, size ) )
return( FALSE );
memcpy( dbuf->data + dbuf->write_point, data, size );
dbuf->write_point = new_write_point;
dbuf->write_point += size;
return( TRUE );
}
@ -110,7 +151,7 @@ vips_dbuf_appendf( VipsDbuf *dbuf, const char *fmt, ... )
line = g_strdup_vprintf( fmt, ap );
va_end( ap );
if( vips_dbuf_append( dbuf, line, strlen( line ) ) ) {
if( vips_dbuf_append( dbuf, (unsigned char *) line, strlen( line ) ) ) {
g_free( line );
return( FALSE );
}
@ -159,12 +200,12 @@ vips_dbuf_destroy( VipsDbuf *dbuf )
*
* Returns: (transfer full): The pointer held by @dbuf.
*/
char *
unsigned char *
vips_dbuf_steal( VipsDbuf *dbuf, size_t *size )
{
char *data;
unsigned char *data;
vips_dbuf_append( dbuf, "", 1 );
vips_dbuf_append( dbuf, (unsigned char *) "", 1 );
dbuf->write_point -= 1;
data = dbuf->data;
@ -180,7 +221,7 @@ vips_dbuf_steal( VipsDbuf *dbuf, size_t *size )
}
/**
* vips_dbuf_data:
* vips_dbuf_string:
* @dbuf: the buffer
* @size: (allow-none): optionally return length in bytes here
*
@ -191,10 +232,10 @@ vips_dbuf_steal( VipsDbuf *dbuf, size_t *size )
*
* Returns: (transfer none): The pointer held by @dbuf.
*/
char *
unsigned char *
vips_dbuf_string( VipsDbuf *dbuf, size_t *size )
{
vips_dbuf_append( dbuf, "", 1 );
vips_dbuf_append( dbuf, (unsigned char *) "", 1 );
dbuf->write_point -= 1;
if( size )

24
libvips/iofuncs/vips.c
View File

@ -619,7 +619,8 @@ parser_element_end_handler( void *user_data, const XML_Char *name )
if( vep->header ) {
if( strcmp( name, "Hist" ) == 0 )
set_history( vep->image,
vips_dbuf_string( &vep->dbuf, NULL ) );
(char *) vips_dbuf_string( &vep->dbuf,
NULL ) );
}
else {
GType gtype = g_type_from_name( vep->type );
@ -631,7 +632,8 @@ parser_element_end_handler( void *user_data, const XML_Char *name )
VIPS_TYPE_SAVE_STRING, gtype ) &&
set_meta( vep->image,
gtype, vep->name,
vips_dbuf_string( &vep->dbuf, NULL ) ) )
(char *) vips_dbuf_string( &vep->dbuf,
NULL ) ) )
vep->error = TRUE;
}
}
@ -646,7 +648,7 @@ parser_data_handler( void *user_data, const XML_Char *data, int len )
printf( "parser_data_handler: %d bytes\n", len );
#endif /*DEBUG*/
vips_dbuf_append( &vep->dbuf, data, len );
vips_dbuf_append( &vep->dbuf, (unsigned char *) data, len );
}
/* Called at the end of vips open ... get any XML after the pixel data
@ -724,9 +726,9 @@ dbuf_append_quotes( VipsDbuf *dbuf, const char *str )
for( p = str; *p; p += len ) {
len = strcspn( p, "\"" );
vips_dbuf_append( dbuf, p, len );
vips_dbuf_append( dbuf, (unsigned char *) p, len );
if( p[len] == '"' )
vips_dbuf_append( dbuf, "\\", 1 );
vips_dbuf_appendf( dbuf, "\\" );
}
}
@ -741,20 +743,20 @@ dbuf_append_amp( VipsDbuf *dbuf, const char *str )
for( p = str; *p; p += len ) {
len = strcspn( p, "&<>" );
vips_dbuf_append( dbuf, p, len );
vips_dbuf_append( dbuf, (unsigned char *) p, len );
switch( p[len] ) {
case '&':
vips_dbuf_append( dbuf, "&amp;", 5 );
vips_dbuf_appendf( dbuf, "&amp;" );
len += 1;
break;
case '<':
vips_dbuf_append( dbuf, "&lt;", 4 );
vips_dbuf_appendf( dbuf, "&lt;" );
len += 1;
break;
case '>':
vips_dbuf_append( dbuf, "&gt;", 4 );
vips_dbuf_appendf( dbuf, "&gt;" );
len += 1;
break;
@ -833,7 +835,7 @@ build_xml( VipsImage *image )
vips_dbuf_appendf( &dbuf, " </meta>\n" );
vips_dbuf_appendf( &dbuf, "</root>\n" );
return( vips_dbuf_steal( &dbuf, NULL ) );
return( (char *) vips_dbuf_steal( &dbuf, NULL ) );
}
static void *
@ -906,7 +908,7 @@ vips__xml_properties( VipsImage *image )
vips_dbuf_appendf( &dbuf, " </properties>\n" );
vips_dbuf_appendf( &dbuf, "</image>\n" );
return( vips_dbuf_steal( &dbuf, NULL ) );
return( (char *) vips_dbuf_steal( &dbuf, NULL ) );
}
/* Append XML to output fd.