sergiotarxz/libvips
sergiotarxz/libvips
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

kinder, gentler underflow check

Browse Source
This commit is contained in:
John Cupitt 2016-07-01 15:30:07 +01:00
parent d2aeec56b5
commit 9442c225ba
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
libvips/convolution/convi.c
View File

@ -833,6 +833,7 @@ intize_to_fixed_point( VipsImage *in, int *out )
double scale; double scale;
int ne; int ne;
double *scaled; double *scaled;
double total_error;
int i; int i;
if( vips_check_matrix( "vips2imask", in, &t ) ) if( vips_check_matrix( "vips2imask", in, &t ) )
@ -864,12 +865,21 @@ intize_to_fixed_point( VipsImage *in, int *out )
/* The smallest coefficient we can manage is 1/64th, we'll just turn /* The smallest coefficient we can manage is 1/64th, we'll just turn
* that into zero. * that into zero.
*
* Find the total error we'll get by rounding down to zero and bail if
* it's significant.
*/ */
total_error = 0.0;
for( i = 0; i < ne; i++ ) for( i = 0; i < ne; i++ )
if( scaled[i] != 0.0 && if( scaled[i] != 0.0 &&
VIPS_FABS( scaled[i] ) < 1.0 / FIXED_SCALE ) { VIPS_FABS( scaled[i] ) < 1.0 / FIXED_SCALE )
total_error += VIPS_FABS( scaled[i] );
/* 0.1 is a 10% error.
*/
if( total_error > 0.1 ) {
#ifdef DEBUG_COMPILE #ifdef DEBUG_COMPILE
printf( "intize_to_fixed_point: underflow\n" ); printf( "intize_to_fixed_point: too many underflows\n" );
#endif /*DEBUG_COMPILE*/ #endif /*DEBUG_COMPILE*/
return( -1 ); return( -1 );