sergiotarxz
/
libvips
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

make pipe read limit configurable 

We had a 1gb limit on the amount of data we would read from a pipe
before giving up.

This patch adds vips_pipe_read_limit_set() and makes this limit
configurable.

See: https://github.com/libvips/libvips/issues/1540
This commit is contained in:
John Cupitt 2020-02-03 16:57:10 +00:00
parent 559ae542ac
commit cf5cad2b3e
4 changed files with 76 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libvips/include/vips/connection.h
View File

@ -89,6 +89,8 @@ GType vips_connection_get_type( void );
const char *vips_connection_filename( VipsConnection *connection );
const char *vips_connection_nick( VipsConnection *connection );
void vips_pipe_read_limit_set( size_t limit );
#define VIPS_TYPE_SOURCE (vips_source_get_type())
#define VIPS_SOURCE( obj ) \
(G_TYPE_CHECK_INSTANCE_CAST( (obj), \

16
libvips/iofuncs/init.c
View File

@ -127,6 +127,8 @@ int vips__leak = 0;
GQuark vips__image_pixels_quark = 0;
#endif /*DEBUG_LEAK*/
static gint64 vips_pipe_read_limit = 1024 * 1024 * 1024;
/**
* vips_get_argv0:
*
@ -428,19 +430,18 @@ vips_init( const char *argv0 )
g_free( locale );
bind_textdomain_codeset( GETTEXT_PACKAGE, "UTF-8" );
/* Deprecated, this is just for compat.
*/
if( g_getenv( "VIPS_INFO" ) ||
g_getenv( "IM_INFO" ) )
vips_info_set( TRUE );
if( g_getenv( "VIPS_PROFILE" ) )
vips_profile_set( TRUE );
/* Default various settings from env.
*/
if( g_getenv( "VIPS_TRACE" ) )
vips_cache_set_trace( TRUE );
if( g_getenv( "VIPS_PIPE_READ_LIMIT" ) )
vips_pipe_read_limit =
g_ascii_strtoull( g_getenv( "VIPS_PIPE_READ_LIMIT" ),
NULL, 10 );
vips_pipe_read_limit_set( vips_pipe_read_limit );
/* Register base vips types.
*/
@ -815,6 +816,9 @@ static GOptionEntry option_entries[] = {
{ "vips-version", 0, G_OPTION_FLAG_NO_ARG,
G_OPTION_ARG_CALLBACK, (gpointer) &vips_lib_version_cb,
N_( "print libvips version" ), NULL },
{ "vips-pipe-read-limit", 0, 0,
G_OPTION_ARG_INT64, (gpointer) &vips_pipe_read_limit,
N_( "read at most this many bytes from a pipe" ), NULL },
{ NULL }
};

10
libvips/iofuncs/region.c
View File

@ -1235,6 +1235,8 @@ vips_region_shrink_uncoded_mean( VipsRegion *from,
* IS stable with respect to the initial arrangement of input values
*/
#define SHRINK_TYPE_MEDIAN( TYPE ) { \
int ls = VIPS_REGION_LSKIP( from ); \
\
for( x = 0; x < target->width; x++ ) { \
TYPE *tp = (TYPE *) p; \
TYPE *tp1 = (TYPE *) (p + ls); \
@ -1265,6 +1267,8 @@ vips_region_shrink_uncoded_mean( VipsRegion *from,
* IS stable with respect to the initial arrangement of input values
*/
#define SHRINK_TYPE_MODE( TYPE ) { \
int ls = VIPS_REGION_LSKIP( from ); \
\
for( x = 0; x < target->width; x++ ) { \
TYPE *tp = (TYPE *) p; \
TYPE *tp1 = (TYPE *) (p + ls); \
@ -1289,6 +1293,8 @@ vips_region_shrink_uncoded_mean( VipsRegion *from,
}
#define SHRINK_TYPE_MAX( TYPE ) { \
int ls = VIPS_REGION_LSKIP( from ); \
\
for( x = 0; x < target->width; x++ ) { \
TYPE *tp = (TYPE *) p; \
TYPE *tp1 = (TYPE *) (p + ls); \
@ -1307,6 +1313,8 @@ vips_region_shrink_uncoded_mean( VipsRegion *from,
}
#define SHRINK_TYPE_MIN( TYPE ) { \
int ls = VIPS_REGION_LSKIP( from ); \
\
for( x = 0; x < target->width; x++ ) { \
TYPE *tp = (TYPE *) p; \
TYPE *tp1 = (TYPE *) (p + ls); \
@ -1327,7 +1335,6 @@ vips_region_shrink_uncoded_mean( VipsRegion *from,
#define SHRINK_TYPE_NEAREST( TYPE ) { \
for( x = 0; x < target->width; x++ ) { \
TYPE *tp = (TYPE *) p; \
TYPE *tp1 = (TYPE *) (p + ls); \
TYPE *tq = (TYPE *) q; \
\
for( z = 0; z < nb; z++ ) \
@ -1343,7 +1350,6 @@ static void \
vips_region_shrink_uncoded_ ## OP( VipsRegion *from, \
VipsRegion *to, const VipsRect *target ) \
{ \
int ls = VIPS_REGION_LSKIP( from ); \
int ps = VIPS_IMAGE_SIZEOF_PEL( from->im ); \
int nb = from->im->Bands; \
\

74
libvips/iofuncs/source.c
View File

@ -1,7 +1,10 @@
/* A byte source/sink .. it can be a pipe, file descriptor, memory area,
* socket, node.js stream, etc.
*
* J.Cupitt, 19/6/14
* 19/6/14
*
* 3/2/20
* - add vips_pipe_read_limit_set()
*/
/*
@ -84,10 +87,40 @@
#define MODE_READWRITE BINARYIZE (O_RDWR)
#define MODE_WRITE BINARYIZE (O_WRONLY | O_CREAT | O_TRUNC)
/* -1 on a pipe isn't actually unbounded. Have a limit to prevent
* huge sources accidentally filling memory.
*
* This can be configured with vips_pipe_read_limit_set().
*/
static size_t vips__pipe_read_limit = 1024 * 1024 * 1024;
/**
* vips_pipe_read_limit_set:
* @limit: maximum number of bytes to buffer from a pipe
*
* If a source does not support mmap or seek and the source is
* used with a loader that can only work from memory, then the data will be
* automatically read into memory to EOF before the loader starts. This can
* produce high memory use if the descriptor represents a large object.
*
* Use vips_pipe_read_limit_set() to limit the size of object that
* will be read in this way. The default is 1GB.
*
* Set a value of -1 to mean no limit.
*
* See also: `--vips-pipe-read-limit` and the environment variable
* `VIPS_PIPE_READ_LIMIT`.
*/
void
vips_pipe_read_limit_set( size_t limit )
{
vips__pipe_read_limit = limit;
}
G_DEFINE_TYPE( VipsSource, vips_source, VIPS_TYPE_CONNECTION );
/* We can't test for seekability or length during _build, since the read and
* seek signal handlers may not have been connected yet. Instead, we test
* seek signal handlers might not have been connected yet. Instead, we test
* when we first need to know.
*/
static int
@ -184,6 +217,9 @@ vips_source_sanity( VipsSource *source )
g_assert( source->length == -1 );
}
else {
/* Something like a seekable file.
*/
/* After we're done with the header, the sniff buffer should
* be gone.
*/
@ -377,6 +413,14 @@ vips_source_new_from_descriptor( int descriptor )
*
* Create an source attached to a file.
*
* If this descriptor does not support mmap and the source is
* used with a loader that can only work from memory, then the data will be
* automatically read into memory to EOF before the loader starts. This can
* produce high memory use if the descriptor represents a large object.
*
* Use vips_pipe_read_limit_set() to limit the size of object that
* will be read in this way. The default is 1GB.
*
* Returns: a new source.
*/
VipsSource *
@ -715,19 +759,14 @@ vips_source_read( VipsSource *source, void *buffer, size_t length )
return( total_read );
}
/* -1 on a pipe isn't actually unbounded. Have a limit to prevent
* huge sources accidentally filling memory.
*
* 1gb. Why not.
*/
static const int vips_pipe_read_limit = 1024 * 1024 * 1024;
/* Read to a position. -1 means read to end of source. Does not change
* read_position.
*/
static int
vips_source_pipe_read_to_position( VipsSource *source, gint64 target )
{
const char *nick = vips_connection_nick( VIPS_CONNECTION( source ) );
gint64 old_read_position;
unsigned char buffer[4096];
@ -742,7 +781,7 @@ vips_source_pipe_read_to_position( VipsSource *source, gint64 target )
(target < 0 ||
(source->length != -1 &&
target > source->length)) ) {
vips_error( vips_connection_nick( VIPS_CONNECTION( source ) ),
vips_error( nick,
_( "bad read to %" G_GINT64_FORMAT ), target );
return( -1 );
}
@ -760,9 +799,9 @@ vips_source_pipe_read_to_position( VipsSource *source, gint64 target )
break;
if( target == -1 &&
source->read_position > vips_pipe_read_limit ) {
vips_error( vips_connection_nick( VIPS_CONNECTION( source ) ),
"%s", _( "pipe too long" ) );
vips__pipe_read_limit != -1 &&
source->read_position > vips__pipe_read_limit ) {
vips_error( nick, "%s", _( "pipe too long" ) );
return( -1 );
}
}
@ -1013,6 +1052,7 @@ vips_source_map_blob( VipsSource *source )
gint64
vips_source_seek( VipsSource *source, gint64 offset, int whence )
{
const char *nick = vips_connection_nick( VIPS_CONNECTION( source ) );
VipsSourceClass *class = VIPS_SOURCE_GET_CLASS( source );
gint64 new_pos;
@ -1039,8 +1079,7 @@ vips_source_seek( VipsSource *source, gint64 offset, int whence )
break;
default:
vips_error( vips_connection_nick( VIPS_CONNECTION( source ) ),
"%s", _( "bad 'whence'" ) );
vips_error( nick, "%s", _( "bad 'whence'" ) );
return( -1 );
}
}
@ -1066,8 +1105,7 @@ vips_source_seek( VipsSource *source, gint64 offset, int whence )
break;
default:
vips_error( vips_connection_nick( VIPS_CONNECTION( source ) ),
"%s", _( "bad 'whence'" ) );
vips_error( nick, "%s", _( "bad 'whence'" ) );
return( -1 );
}
}
@ -1081,7 +1119,7 @@ vips_source_seek( VipsSource *source, gint64 offset, int whence )
if( new_pos < 0 ||
(source->length != -1 &&
new_pos > source->length) ) {
vips_error( vips_connection_nick( VIPS_CONNECTION( source ) ),
vips_error( nick,
_( "bad seek to %" G_GINT64_FORMAT ), new_pos );
return( -1 );
}