started png and jpg with dbuf

rad and webp still to do, maybe tiff

README.md

@@ -46,8 +46,7 @@ Untar, then in the libvips directory you should just be able to do:
 	$ ./configure
 
 Check the summary at the end of `configure` carefully. 
-libvips must have `build-essential`, `pkg-config`, `glib2.0-dev`, and 
+libvips must have `build-essential`, `pkg-config`, `glib2.0-dev`.
-`libxml2-dev`. 
 
 For the vips8 Python binding, you must have 
 `gobject-introspection`, `python-gi-dev`, and `libgirepository1.0-dev`.
@@ -145,8 +144,7 @@ Static analysis with:
 
 # Dependencies 
 
-libvips has to have `gettext`, `glib2.0-dev` and `libxml2-dev`. Other
+libvips has to have `glib2.0-dev`. Other dependencies are optional, see below.
-dependencies are optional, see below.
 
 # Optional dependencies
 

TODO

@@ -1,5 +1,25 @@
 - use VipsDbuf for tiffsave_buffer etc.
 
+	$ grep -l save_buffer *.c
+	dzsave.c
+	jpegsave.c
+	pngsave.c
+	radsave.c
+	tiffsave.c
+	webpsave.c
+
+  done png, jpg
+
+  tiff needs seek ... perhaps add this to dbuf?
+
+  dzsave uses gsf
+
+  rad and webp left to do
+
+  vips2jpeg term_destination() needs some kind of truncate call for dbuf
+
+
+
 - verify xml data against master for vips save and dzsave
 
 

libvips/foreign/vips2jpeg.c

@@ -82,6 +82,8 @@
  * 	- turn off chroma subsample for Q > 90
  * 7/11/16
  * 	- move exif handling out to exif.c
+ * 27/2/17
+ * 	- use dbuf for memory output
  */
 
 /*
@@ -670,112 +672,6 @@ vips__jpeg_write_file( VipsImage *in,
 	return( 0 );
 }
 
-/* We can't predict how large the output buffer we need is, because we might
- * need space for ICC profiles and stuff. So we write to a linked list of mem
- * buffers and add a new one as they fill.
- */
-
-#define BUFFER_SIZE (10000)
-
-/* A buffer.
- */
-typedef struct _Block {
-	j_compress_ptr cinfo;
-
-	struct _Block *first;
-	struct _Block *next;
-
-	JOCTET *data;		/* Allocated area */
-	size_t size;		/* Max size */
-	size_t used;		/* How much has been used */
-} Block;
-
-static Block *
-block_new( j_compress_ptr cinfo )
-{
-	Block *block;
-
-	block = (Block *) (*cinfo->mem->alloc_large) 
-		( (j_common_ptr) cinfo, JPOOL_IMAGE, sizeof( Block ) );
-
-	block->cinfo = cinfo;
-	block->first = block;
-	block->next = NULL;
-	block->data = (JOCTET *) (*cinfo->mem->alloc_large) 
-		( (j_common_ptr) cinfo, JPOOL_IMAGE, BUFFER_SIZE );
-	block->size = BUFFER_SIZE;
-	block->used = 0;
-
-	return( block );
-}
-
-static Block *
-block_last( Block *block )
-{
-	while( block->next )
-		block = block->next;
-
-	return( block );
-}
-
-static Block *
-block_append( Block *block )
-{
-	Block *new;
-
-	g_assert( block );
-
-	new = block_new( block->cinfo );
-	new->first = block->first;
-	block_last( block )->next = new;
-
-	return( new );
-}
-
-static size_t
-block_length( Block *block )
-{
-	size_t len;
-
-	len = 0;
-	for( block = block->first; block; block = block->next )
-		len += block->used;
-
-	return( len );
-}
-
-static void
-block_copy( Block *block, void *dest )
-{
-	JOCTET *p;
-	
-	p = dest;
-	for( block = block->first; block; block = block->next ) {
-		memcpy( p, block->data, block->used );
-		p += block->used;
-	}
-}
-
-#ifdef DEBUG
-static void
-block_print( Block *block )
-{
-	int i;
-
-	printf( "total length = %zd\n", block_length( block ) );
-	printf( "set of blocks:\n" );
-
-	i = 0;
-	for( block = block->first; block; block = block->next ) {
-		printf( "%d) %p, first = %p, next = %p"
-			"\t data = %p, size = %zd, used = %zd\n", 
-			i, block, block->first, block->next,
-			block->data, block->size, block->used );
-		i += 1;
-	}
-}
-#endif /*DEBUG*/
-
 /* Just like the above, but we write to a memory buffer.
  *
  * A memory buffer for the compressed image.
@@ -788,16 +684,36 @@ typedef struct {
 	/* Private stuff during write.
 	 */
 
-	/* Build the output area here in chunks.
+	/* Build the output area here.
 	 */
-	Block *block;
+	VipsDbuf dbuf;
 
-	/* Copy the compressed area here.
+	/* Write the generated area here.
 	 */
 	void **obuf;		/* Allocated buffer, and size */
 	size_t *olen;
 } OutputBuffer;
 
+/* Buffer full method ... allocate a new output block. This is only called 
+ * when the output area is exactly full.
+ */
+METHODDEF(boolean)
+empty_output_buffer( j_compress_ptr cinfo )
+{
+	OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
+
+	size_t size;
+
+	vips_dbuf_allocate( &buf->dbuf, 10000 ); 
+	buf->pub.next_output_byte = 
+		(JOCTET *) vips_dbuf_get_write( &buf->dbuf, &size );
+	buf->pub.free_in_buffer = size;
+
+	/* TRUE means we've made some more space.
+	 */
+	return( 1 );
+}
+
 /* Init dest method.
  */
 METHODDEF(void)
@@ -805,38 +721,8 @@ init_destination( j_compress_ptr cinfo )
 {
 	OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
 
-	/* Allocate relative to the image we are writing .. freed when we junk
+	vips_dbuf_init( &buf->dbuf ); 
-	 * this output.
+	empty_output_buffer( cinfo ); 
-	 */
-	buf->block = block_new( cinfo );
-
-	/* Set buf pointers for library.
-	 */
-	buf->pub.next_output_byte = buf->block->data;
-	buf->pub.free_in_buffer = buf->block->size;
-}
-
-/* Buffer full method ... allocate a new output block.
- */
-METHODDEF(boolean)
-empty_output_buffer( j_compress_ptr cinfo )
-{
-	OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
-
-	/* Record how many bytes we used. empty_output_buffer() is always
-	 * called when the buffer is exactly full.
-	 */
-	buf->block->used = buf->block->size;
-
-	/* New block and reset.
-	 */
-	buf->block = block_append( buf->block );
-	buf->pub.next_output_byte = buf->block->data;
-	buf->pub.free_in_buffer = buf->block->size;
-
-	/* TRUE means we've made some more space.
-	 */
-	return( 1 );
 }
 
 /* Cleanup. Copy the set of blocks out as a big lump.
@@ -846,35 +732,15 @@ term_destination( j_compress_ptr cinfo )
 {
         OutputBuffer *buf = (OutputBuffer *) cinfo->dest;
 
-	size_t len;
+	size_t size;
-	void *obuf;
 
-	/* Record the number of bytes we wrote in the final buffer.
+	/* This can be called before the output area fills. We need to chop
-	 * pub.free_in_buffer is valid here.
+	 * size down to the number of bytes actually written.
 	 */
-	buf->block->used = buf->block->size - buf->pub.free_in_buffer;
+	buf->dbuf.write_point = buf->dbuf.max_size - buf->pub.free_in_buffer;
 
-#ifdef DEBUG
+	*(buf->obuf) = vips_dbuf_steal( &buf->dbuf, &size );
-	block_print( buf->block );
+	*(buf->olen) = size; 
-#endif /*DEBUG*/
-
-	/* ... and we can count up our buffers now.
-	 */
-	len = block_length( buf->block );
-
-	/* Allocate and copy to the output area.
-	 */
-	if( !(obuf = vips_malloc( NULL, len )) )
-		ERREXIT( cinfo, JERR_FILE_WRITE );
-	else {
-		/* coverity doesn't know ERREXIT() does not return, so put
-		 * this in an else.
-		 */
-		*(buf->obuf) = obuf;
-		*(buf->olen) = len;
-
-		block_copy( buf->block, obuf );
-	}
 }
 
 /* Set dest to one of our objects.

libvips/foreign/vipspng.c

@@ -59,6 +59,8 @@
  * 	- support --strip option
  * 17/1/17
  * 	- invalidate operation on read error
+ * 27/2/17
+ * 	- use dbuf for buffer output
  */
 
 /*
@@ -734,10 +736,7 @@ typedef struct {
 	VipsImage *memory;
 
 	FILE *fp;
-
+	VipsDbuf dbuf;
-	char *buf;
-	size_t len;
-	size_t alloc;
 
 	png_structp pPng;
 	png_infop pInfo;
@@ -749,7 +748,7 @@ write_finish( Write *write )
 {
 	VIPS_FREEF( fclose, write->fp );
 	VIPS_UNREF( write->memory );
-	VIPS_FREE( write->buf );
+	vips_dbuf_destroy( &write->dbuf );
 	if( write->pPng )
 		png_destroy_write_struct( &write->pPng, &write->pInfo );
 }
@@ -771,9 +770,7 @@ write_new( VipsImage *in )
 	write->in = in;
 	write->memory = NULL;
 	write->fp = NULL;
-	write->buf = NULL;
+	vips_dbuf_init( &write->dbuf );
-	write->len = 0;
-	write->alloc = 0;
 	g_signal_connect( in, "close", 
 		G_CALLBACK( write_destroy ), write ); 
 
@@ -1014,41 +1011,12 @@ vips__png_write( VipsImage *in, const char *filename,
 	return( 0 );
 }
 
-static void
-write_grow( Write *write, size_t grow_len )
-{
-	size_t new_len = write->len + grow_len;
-
-	if( new_len > write->alloc ) {
-		size_t proposed_alloc = (16 + write->alloc) * 3 / 2;
-
-		write->alloc = VIPS_MAX( proposed_alloc, new_len );
-
-		/* Our result must be freedd with g_free(), so it's OK to use
-		 * g_realloc(). 
-		 */
-	 	write->buf = g_realloc( write->buf, write->alloc );
-
-		VIPS_DEBUG_MSG( "write_buf_grow: grown to %zd bytes\n",
-			write->alloc );
-	}
-}
-
 static void
 user_write_data( png_structp png_ptr, png_bytep data, png_size_t length )
 {
 	Write *write = (Write *) png_get_io_ptr( png_ptr );
 
-	char *write_start;
+	vips_dbuf_append( &write->dbuf, data, length ); 
-
-	write_grow( write, length );
-
-	write_start = write->buf + write->len;
-	memcpy( write_start, data, length );
-
-	write->len += length;
-
-	g_assert( write->len <= write->alloc );
 }
 
 int
@@ -1073,10 +1041,7 @@ vips__png_write_buf( VipsImage *in,
 		return( -1 );
 	}
 
-	*obuf = write->buf;
+	*obuf = vips_dbuf_steal( &write->dbuf, olen );
-	write->buf = NULL;
-	if( olen )
-		*olen = write->len;
 
 	write_finish( write );
 

libvips/include/vips/dbuf.h

@@ -48,7 +48,7 @@ typedef struct _VipsDbuf {
 
 	/* The current base, and the size of the allocated memory area.
 	 */
-	char *data;
+	unsigned char *data;
 	size_t max_size;
 
 	/* And the write point.
@@ -58,12 +58,15 @@ typedef struct _VipsDbuf {
 
 void vips_dbuf_destroy( VipsDbuf *buf );
 void vips_dbuf_init( VipsDbuf *buf );
-gboolean vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size );
+gboolean vips_dbuf_allocate( VipsDbuf *dbuf, size_t size );
+unsigned char *vips_dbuf_get_write( VipsDbuf *dbuf, size_t *size );
+gboolean vips_dbuf_append( VipsDbuf *dbuf, 
+	const unsigned char *data, size_t size );
 gboolean vips_dbuf_appendf( VipsDbuf *dbuf, const char *fmt, ... );
 void vips_dbuf_rewind( VipsDbuf *dbuf );
 void vips_dbuf_destroy( VipsDbuf *dbuf );
-char *vips_dbuf_string( VipsDbuf *dbuf, size_t *size );
+unsigned char *vips_dbuf_string( VipsDbuf *dbuf, size_t *size );
-char *vips_dbuf_steal( VipsDbuf *dbuf, size_t *size );
+unsigned char *vips_dbuf_steal( VipsDbuf *dbuf, size_t *size );
 
 #endif /*VIPS_DBUF_H*/
 

libvips/iofuncs/dbuf.c

@@ -56,24 +56,23 @@ vips_dbuf_init( VipsDbuf *dbuf )
 }
 
 /**
- * vips_dbuf_append:
+ * vips_dbuf_allocate:
  * @dbuf: the buffer
- * @data: the data to append to the buffer
+ * @size: the size to allocate
- * @size: the size of the len to append
  *
- * Append len bytes from @data to the buffer. The buffer expands if necessary. 
+ * Make sure @dbuf has at least @size bytes available for writing.
  * 
  * Returns: %FALSE on out of memory, %TRUE otherwise.
  */
 gboolean
-vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size )
+vips_dbuf_allocate( VipsDbuf *dbuf, size_t size )
 {
 	size_t new_write_point = dbuf->write_point + size;
 
 	if( new_write_point > dbuf->max_size ) { 
 		size_t new_max_size = 3 * (16 + new_write_point) / 2;
 
-		char *new_data;
+		unsigned char *new_data;
 
 		if( !(new_data = g_try_realloc( dbuf->data, new_max_size )) ) {
 			vips_error( "VipsDbuf", "%s", _( "out of memory" ) );
@@ -84,8 +83,50 @@ vips_dbuf_append( VipsDbuf *dbuf, const char *data, size_t size )
 		dbuf->max_size = new_max_size;
 	}
 
+	return( TRUE ); 
+}
+
+/**
+ * vips_dbuf_get_write:
+ * @dbuf: the buffer
+ * @size: (allow-none): optionally return length in bytes here
+ *
+ * Return a pointer to an area you can write to, return length of area in
+ * @size. Use vips_dbuf_allocate() before this call to make the space.
+ * 
+ * Returns: (transfer none): start of write area.
+ */
+unsigned char *
+vips_dbuf_get_write( VipsDbuf *dbuf, size_t *size )
+{
+	unsigned char *data = dbuf->data + dbuf->write_point;
+
+	if( size )
+		*size = dbuf->max_size - dbuf->write_point;
+
+	dbuf->write_point = dbuf->max_size;
+
+	return( data ); 
+}
+
+/**
+ * vips_dbuf_append:
+ * @dbuf: the buffer
+ * @data: the data to append to the buffer
+ * @size: the size of the len to append
+ *
+ * Append len bytes from @data to the buffer. The buffer expands if necessary. 
+ * 
+ * Returns: %FALSE on out of memory, %TRUE otherwise.
+ */
+gboolean
+vips_dbuf_append( VipsDbuf *dbuf, const unsigned char *data, size_t size )
+{
+	if( !vips_dbuf_allocate( dbuf, size ) )
+		return( FALSE ); 
+
 	memcpy( dbuf->data + dbuf->write_point, data, size );
-	dbuf->write_point = new_write_point;
+	dbuf->write_point += size;
 
 	return( TRUE ); 
 }
@@ -110,7 +151,7 @@ vips_dbuf_appendf( VipsDbuf *dbuf, const char *fmt, ... )
 	line = g_strdup_vprintf( fmt, ap ); 
         va_end( ap );
 
-	if( vips_dbuf_append( dbuf, line, strlen( line ) ) ) {
+	if( vips_dbuf_append( dbuf, (unsigned char *) line, strlen( line ) ) ) {
 		g_free( line ); 
 		return( FALSE );
 	}
@@ -159,12 +200,12 @@ vips_dbuf_destroy( VipsDbuf *dbuf )
  *
  * Returns: (transfer full): The pointer held by @dbuf.
  */
-char *
+unsigned char *
 vips_dbuf_steal( VipsDbuf *dbuf, size_t *size )
 {
-	char *data;
+	unsigned char *data;
 
-	vips_dbuf_append( dbuf, "", 1 );
+	vips_dbuf_append( dbuf, (unsigned char *) "", 1 );
 	dbuf->write_point -= 1;
 
 	data = dbuf->data;
@@ -180,7 +221,7 @@ vips_dbuf_steal( VipsDbuf *dbuf, size_t *size )
 }
 
 /**
- * vips_dbuf_data:
+ * vips_dbuf_string:
  * @dbuf: the buffer
  * @size: (allow-none): optionally return length in bytes here
  *
@@ -191,10 +232,10 @@ vips_dbuf_steal( VipsDbuf *dbuf, size_t *size )
  *
  * Returns: (transfer none): The pointer held by @dbuf.
  */
-char *
+unsigned char *
 vips_dbuf_string( VipsDbuf *dbuf, size_t *size )
 {
-	vips_dbuf_append( dbuf, "", 1 );
+	vips_dbuf_append( dbuf, (unsigned char *) "", 1 );
 	dbuf->write_point -= 1;
 
 	if( size )

libvips/iofuncs/vips.c

@@ -619,7 +619,8 @@ parser_element_end_handler( void *user_data, const XML_Char *name )
 		if( vep->header ) {
 			if( strcmp( name, "Hist" ) == 0 ) 
 				set_history( vep->image, 
-					vips_dbuf_string( &vep->dbuf, NULL ) ); 
+					(char *) vips_dbuf_string( &vep->dbuf, 
+						NULL ) ); 
 		}
 		else {
 			GType gtype = g_type_from_name( vep->type );
@@ -631,7 +632,8 @@ parser_element_end_handler( void *user_data, const XML_Char *name )
 					VIPS_TYPE_SAVE_STRING, gtype ) &&
 				set_meta( vep->image, 
 					gtype, vep->name, 
-					vips_dbuf_string( &vep->dbuf, NULL ) ) )
+					(char *) vips_dbuf_string( &vep->dbuf, 
+						NULL ) ) )
 				vep->error = TRUE;
 		}
 	}
@@ -646,7 +648,7 @@ parser_data_handler( void *user_data, const XML_Char *data, int len )
 	printf( "parser_data_handler: %d bytes\n", len ); 
 #endif /*DEBUG*/
 
-	vips_dbuf_append( &vep->dbuf, data, len );
+	vips_dbuf_append( &vep->dbuf, (unsigned char *) data, len );
 }
 
 /* Called at the end of vips open ... get any XML after the pixel data
@@ -724,9 +726,9 @@ dbuf_append_quotes( VipsDbuf *dbuf, const char *str )
 	for( p = str; *p; p += len ) {
 		len = strcspn( p, "\"" );
 
-		vips_dbuf_append( dbuf, p, len );
+		vips_dbuf_append( dbuf, (unsigned char *) p, len );
 		if( p[len] == '"' )
-			vips_dbuf_append( dbuf, "\\", 1 );
+			vips_dbuf_appendf( dbuf, "\\" );
 	}
 }
 
@@ -741,20 +743,20 @@ dbuf_append_amp( VipsDbuf *dbuf, const char *str )
 	for( p = str; *p; p += len ) {
 		len = strcspn( p, "&<>" );
 
-		vips_dbuf_append( dbuf, p, len );
+		vips_dbuf_append( dbuf, (unsigned char *) p, len );
 		switch( p[len] ) {
 		case '&': 
-			vips_dbuf_append( dbuf, "&amp;", 5 );
+			vips_dbuf_appendf( dbuf, "&amp;" );
 			len += 1;
 			break;
 
 		case '<': 
-			vips_dbuf_append( dbuf, "&lt;", 4 );
+			vips_dbuf_appendf( dbuf, "&lt;" );
 			len += 1;
 			break;
 
 		case '>': 
-			vips_dbuf_append( dbuf, "&gt;", 4 );
+			vips_dbuf_appendf( dbuf, "&gt;" );
 			len += 1;
 			break;
 
@@ -833,7 +835,7 @@ build_xml( VipsImage *image )
 	vips_dbuf_appendf( &dbuf, "  </meta>\n" );  
 	vips_dbuf_appendf( &dbuf, "</root>\n" );  
 
-	return( vips_dbuf_steal( &dbuf, NULL ) ); 
+	return( (char *) vips_dbuf_steal( &dbuf, NULL ) ); 
 }
 
 static void *
@@ -906,7 +908,7 @@ vips__xml_properties( VipsImage *image )
 	vips_dbuf_appendf( &dbuf, "  </properties>\n" );  
 	vips_dbuf_appendf( &dbuf, "</image>\n" );  
 
-	return( vips_dbuf_steal( &dbuf, NULL ) ); 
+	return( (char *) vips_dbuf_steal( &dbuf, NULL ) ); 
 }
 
 /* Append XML to output fd.