better smartcrop
attention mode now centres on objects, rather than discarding non-objects
This commit is contained in:
John Cupitt
parent
fd6006bacb
commit
2ef1896fd0
@ -6,6 +6,8 @@
|
||||
*
|
||||
* 1/3/17
|
||||
* - first version, from sharp
|
||||
* 14/3/17
|
||||
* - revised attention smartcrop
|
||||
*/
|
||||
|
||||
/*
|
||||
@ -63,97 +65,180 @@ typedef struct _VipsSmartcrop {
|
||||
int height;
|
||||
VipsInteresting interesting;
|
||||
|
||||
VipsImage *sobel;
|
||||
VipsImage *sobel90;
|
||||
|
||||
} VipsSmartcrop;
|
||||
|
||||
typedef VipsConversionClass VipsSmartcropClass;
|
||||
|
||||
G_DEFINE_TYPE( VipsSmartcrop, vips_smartcrop, VIPS_TYPE_CONVERSION );
|
||||
|
||||
static void
|
||||
vips_smartcrop_dispose( GObject *gobject )
|
||||
static int
|
||||
vips_smartcrop_score( VipsSmartcrop *smartcrop, VipsImage *in,
|
||||
int left, int top, int width, int height, double *score )
|
||||
{
|
||||
VipsSmartcrop *smartcrop = (VipsSmartcrop *) gobject;
|
||||
VipsImage **t = (VipsImage **)
|
||||
vips_object_local_array( VIPS_OBJECT( smartcrop ), 2 );
|
||||
|
||||
VIPS_UNREF( smartcrop->sobel );
|
||||
VIPS_UNREF( smartcrop->sobel90 );
|
||||
if( vips_extract_area( in, &t[0], left, top, width, height, NULL ) ||
|
||||
vips_hist_find( t[0], &t[1], NULL ) ||
|
||||
vips_hist_entropy( t[1], score, NULL ) )
|
||||
return( -1 );
|
||||
|
||||
G_OBJECT_CLASS( vips_smartcrop_parent_class )->dispose( gobject );
|
||||
return( 0 );
|
||||
}
|
||||
|
||||
/* Entropy-style smartcrop. Repeatedly discard low interest areas. This should
|
||||
* be faster for very large images.
|
||||
*/
|
||||
static int
|
||||
vips_smartcrop_entropy( VipsSmartcrop *smartcrop,
|
||||
VipsImage *in, int *left, int *top )
|
||||
{
|
||||
int max_slice_size;
|
||||
int width;
|
||||
int height;
|
||||
|
||||
*left = 0;
|
||||
*top = 0;
|
||||
width = in->Xsize;
|
||||
height = in->Ysize;
|
||||
|
||||
/* How much do we trim by each iteration? Aim for 8 steps in the axis
|
||||
* that needs trimming most.
|
||||
*/
|
||||
max_slice_size = VIPS_MAX(
|
||||
ceil( (width - smartcrop->width) / 8.0 ),
|
||||
ceil( (height - smartcrop->height) / 8.0 ) );
|
||||
|
||||
/* Repeatedly take a slice off width and height until we
|
||||
* reach the target.
|
||||
*/
|
||||
while( width > smartcrop->width ||
|
||||
height > smartcrop->height ) {
|
||||
const int slice_width =
|
||||
VIPS_MIN( width - smartcrop->width, max_slice_size );
|
||||
const int slice_height =
|
||||
VIPS_MIN( height - smartcrop->height, max_slice_size );
|
||||
|
||||
if( slice_width > 0 ) {
|
||||
double left_score;
|
||||
double right_score;
|
||||
|
||||
if( vips_smartcrop_score( smartcrop, in,
|
||||
*left, *top, slice_width, height, &left_score ) )
|
||||
return( -1 );
|
||||
|
||||
if( vips_smartcrop_score( smartcrop, in,
|
||||
*left + width - slice_width, *top,
|
||||
slice_width, height, &right_score ) )
|
||||
return( -1 );
|
||||
|
||||
width -= slice_width;
|
||||
if( left_score < right_score )
|
||||
*left += slice_width;
|
||||
}
|
||||
|
||||
if( slice_height > 0 ) {
|
||||
double top_score;
|
||||
double bottom_score;
|
||||
|
||||
if( vips_smartcrop_score( smartcrop, in,
|
||||
*left, *top, width, slice_height, &top_score ) )
|
||||
return( -1 );
|
||||
|
||||
if( vips_smartcrop_score( smartcrop, in,
|
||||
*left, *top + height - slice_height,
|
||||
width, slice_height, &bottom_score ) )
|
||||
return( -1 );
|
||||
|
||||
height -= slice_height;
|
||||
if( top_score < bottom_score )
|
||||
*top += slice_height;
|
||||
}
|
||||
}
|
||||
|
||||
return( 0 );
|
||||
}
|
||||
|
||||
static int
|
||||
vips_smartcrop_score( VipsSmartcrop *smartcrop, VipsImage *image, double *score )
|
||||
vips_smartcrop_attention( VipsSmartcrop *smartcrop,
|
||||
VipsImage *in, int *left, int *top )
|
||||
{
|
||||
VipsImage **t = (VipsImage **)
|
||||
vips_object_local_array( VIPS_OBJECT( smartcrop ), 20 );
|
||||
|
||||
/* ab ranges for skin colours. Trained with http://humanae.tumblr.com/
|
||||
*/
|
||||
double ab_low[2] = { 3.0, 4.0 };
|
||||
double ab_high[2] = { 22.0, 31.0 };
|
||||
static double ab_low[2] = { 3.0, 4.0 };
|
||||
static double ab_high[2] = { 22.0, 31.0 };
|
||||
|
||||
switch( smartcrop->interesting ) {
|
||||
case VIPS_INTERESTING_ENTROPY:
|
||||
if( vips_hist_find( image, &t[0], NULL ) ||
|
||||
vips_hist_entropy( t[0], score, NULL ) )
|
||||
return( -1 );
|
||||
break;
|
||||
VipsImage **t = (VipsImage **)
|
||||
vips_object_local_array( VIPS_OBJECT( smartcrop ), 23 );
|
||||
|
||||
case VIPS_INTERESTING_ATTENTION:
|
||||
/* Convert to LAB and just use the first three bands.
|
||||
*/
|
||||
if( vips_colourspace( image, &t[0],
|
||||
VIPS_INTERPRETATION_LAB, NULL ) ||
|
||||
vips_extract_band( t[0], &t[1], 0, "n", 3, NULL ) )
|
||||
return( -1 );
|
||||
int hshrink;
|
||||
int vshrink;
|
||||
double max;
|
||||
int x_pos;
|
||||
int y_pos;
|
||||
|
||||
/* Sobel edge-detect on L.
|
||||
*/
|
||||
if( vips_extract_band( t[1], &t[2], 0, NULL ) ||
|
||||
vips_conv( t[2], &t[3], smartcrop->sobel, NULL ) ||
|
||||
vips_conv( t[2], &t[4], smartcrop->sobel90, NULL ) ||
|
||||
vips_abs( t[3], &t[5], NULL ) ||
|
||||
vips_abs( t[4], &t[6], NULL ) ||
|
||||
vips_add( t[5], t[6], &t[7], NULL ))
|
||||
return( -1 );
|
||||
if( !(t[21] = vips_image_new_matrixv( 3, 3,
|
||||
-1.0, 0.0, 1.0, -2.0, 0.0, 2.0, -1.0, 0.0, 1.0 )) )
|
||||
return( -1 );
|
||||
if( vips_rot( t[21], &t[22], VIPS_ANGLE_D90, NULL ) )
|
||||
return( -1 );
|
||||
|
||||
/* Look for skin colours, plus L > 15.
|
||||
*/
|
||||
if( vips_extract_band( t[1], &t[8], 1, "n", 2, NULL ) ||
|
||||
vips_moreeq_const( t[8], &t[9], ab_low, 2, NULL ) ||
|
||||
vips_lesseq_const( t[8], &t[10], ab_high, 2, NULL ) ||
|
||||
vips_andimage( t[9], t[10], &t[11], NULL ) ||
|
||||
vips_bandand( t[11], &t[12], NULL ) ||
|
||||
vips_moreeq_const1( t[2], &t[18], 15.0, NULL ) ||
|
||||
vips_andimage( t[12], t[18], &t[19], NULL ) )
|
||||
return( -1 );
|
||||
/* Convert to LAB and just use the first three bands.
|
||||
*/
|
||||
if( vips_colourspace( in, &t[0], VIPS_INTERPRETATION_LAB, NULL ) ||
|
||||
vips_extract_band( t[0], &t[1], 0, "n", 3, NULL ) )
|
||||
return( -1 );
|
||||
|
||||
/* Look for saturated areas.
|
||||
*/
|
||||
if( vips_colourspace( t[1], &t[13],
|
||||
VIPS_INTERPRETATION_LCH, NULL ) ||
|
||||
vips_extract_band( t[13], &t[14], 1, NULL ) ||
|
||||
vips_more_const1( t[14], &t[15], 60.0, NULL ) )
|
||||
return( -1 );
|
||||
/* Sobel edge-detect on L.
|
||||
*/
|
||||
if( vips_extract_band( t[1], &t[2], 0, NULL ) ||
|
||||
vips_conv( t[2], &t[3], t[21], NULL ) ||
|
||||
vips_conv( t[2], &t[4], t[22], NULL ) ||
|
||||
vips_abs( t[3], &t[5], NULL ) ||
|
||||
vips_abs( t[4], &t[6], NULL ) ||
|
||||
vips_add( t[5], t[6], &t[7], NULL ) )
|
||||
return( -1 );
|
||||
|
||||
/* Sum and find max.
|
||||
*/
|
||||
if( vips_add( t[7], t[19], &t[16], NULL ) ||
|
||||
vips_add( t[16], t[15], &t[17], NULL ) ||
|
||||
vips_avg( t[17], score, NULL ) )
|
||||
return( -1 );
|
||||
break;
|
||||
/* Look for skin colours, plus L > 15.
|
||||
*/
|
||||
if( vips_extract_band( t[1], &t[8], 1, "n", 2, NULL ) ||
|
||||
vips_moreeq_const( t[8], &t[9], ab_low, 2, NULL ) ||
|
||||
vips_lesseq_const( t[8], &t[10], ab_high, 2, NULL ) ||
|
||||
vips_andimage( t[9], t[10], &t[11], NULL ) ||
|
||||
vips_bandand( t[11], &t[12], NULL ) ||
|
||||
vips_moreeq_const1( t[2], &t[18], 15.0, NULL ) ||
|
||||
vips_andimage( t[12], t[18], &t[19], NULL ) )
|
||||
return( -1 );
|
||||
|
||||
case VIPS_INTERESTING_CENTRE:
|
||||
case VIPS_INTERESTING_NONE:
|
||||
default:
|
||||
g_assert_not_reached();
|
||||
break;
|
||||
}
|
||||
/* Look for saturated areas.
|
||||
*/
|
||||
if( vips_colourspace( t[1], &t[13],
|
||||
VIPS_INTERPRETATION_LCH, NULL ) ||
|
||||
vips_extract_band( t[13], &t[14], 1, NULL ) ||
|
||||
vips_more_const1( t[14], &t[15], 60.0, NULL ) )
|
||||
return( -1 );
|
||||
|
||||
VIPS_DEBUG_MSG( "vips_smartcrop_score: %g\n", *score );
|
||||
/* Sum, shrink and find maxpos. We could blur too, but the box filter
|
||||
* shrink uses will do more or less the same thing. Shrink to ~ 32x32, it
|
||||
* should give us enough precision for positioning, while also grouping
|
||||
* high-value areas.
|
||||
*/
|
||||
hshrink = ceil( in->Xsize / 32.0 );
|
||||
vshrink = ceil( in->Ysize / 32.0 );
|
||||
if( vips_add( t[7], t[19], &t[16], NULL ) ||
|
||||
vips_add( t[16], t[15], &t[17], NULL ) ||
|
||||
vips_shrink( t[17], &t[20], hshrink, vshrink, NULL ) ||
|
||||
vips_max( t[20], &max, "x", &x_pos, "y", &y_pos, NULL ) )
|
||||
return( -1 );
|
||||
|
||||
/* Centre the crop over the max.
|
||||
*/
|
||||
*left = VIPS_CLIP( 0,
|
||||
hshrink * x_pos - smartcrop->width / 2,
|
||||
in->Xsize - smartcrop->width );
|
||||
*top = VIPS_CLIP( 0,
|
||||
vshrink * y_pos - smartcrop->height / 2,
|
||||
in->Ysize - smartcrop->height );
|
||||
|
||||
return( 0 );
|
||||
}
|
||||
@ -166,12 +251,8 @@ vips_smartcrop_build( VipsObject *object )
|
||||
VipsSmartcrop *smartcrop = (VipsSmartcrop *) object;
|
||||
VipsImage **t = (VipsImage **) vips_object_local_array( object, 2 );
|
||||
|
||||
VipsImage *in;
|
||||
int max_slice_size;
|
||||
int left;
|
||||
int top;
|
||||
int width;
|
||||
int height;
|
||||
|
||||
if( VIPS_OBJECT_CLASS( vips_smartcrop_parent_class )->
|
||||
build( object ) )
|
||||
@ -184,100 +265,25 @@ vips_smartcrop_build( VipsObject *object )
|
||||
return( -1 );
|
||||
}
|
||||
|
||||
if( !(smartcrop->sobel = vips_image_new_matrixv( 3, 3,
|
||||
-1.0, 0.0, 1.0, -2.0, 0.0, 2.0, -1.0, 0.0, 1.0 )) ||
|
||||
vips_rot( smartcrop->sobel, &smartcrop->sobel90,
|
||||
VIPS_ANGLE_D90, NULL ) )
|
||||
return( -1 );
|
||||
|
||||
in = smartcrop->in;
|
||||
left = 0;
|
||||
top = 0;
|
||||
width = in->Xsize;
|
||||
height = in->Ysize;
|
||||
|
||||
/* How much do we trim by each iteration? Aim for 8 steps in the axis
|
||||
* that needs trimming most.
|
||||
*/
|
||||
max_slice_size = VIPS_MAX(
|
||||
ceil( (width - smartcrop->width) / 8.0 ),
|
||||
ceil( (height - smartcrop->height) / 8.0 ) );
|
||||
|
||||
switch( smartcrop->interesting ) {
|
||||
case VIPS_INTERESTING_NONE:
|
||||
break;
|
||||
|
||||
case VIPS_INTERESTING_CENTRE:
|
||||
width = smartcrop->width;
|
||||
height = smartcrop->height;
|
||||
left = (in->Xsize - width) / 2;
|
||||
top = (in->Ysize - height) / 2;
|
||||
left = (smartcrop->in->Xsize - smartcrop->width) / 2;
|
||||
top = (smartcrop->in->Ysize - smartcrop->height) / 2;
|
||||
break;
|
||||
|
||||
case VIPS_INTERESTING_ENTROPY:
|
||||
if( vips_smartcrop_entropy( smartcrop,
|
||||
smartcrop->in, &left, &top ) )
|
||||
return( -1 );
|
||||
break;
|
||||
|
||||
case VIPS_INTERESTING_ATTENTION:
|
||||
/* Repeatedly take a slice off width and height until we
|
||||
* reach the target.
|
||||
*/
|
||||
while( width > smartcrop->width ||
|
||||
height > smartcrop->height ) {
|
||||
const int slice_width =
|
||||
VIPS_MIN( width - smartcrop->width,
|
||||
max_slice_size );
|
||||
const int slice_height =
|
||||
VIPS_MIN( height - smartcrop->height,
|
||||
max_slice_size );
|
||||
|
||||
if( slice_width > 0 ) {
|
||||
VipsImage **t = (VipsImage **)
|
||||
vips_object_local_array( object, 4 );
|
||||
|
||||
double left_score;
|
||||
double right_score;
|
||||
|
||||
if( vips_extract_area( in, &t[0],
|
||||
left, top, slice_width, height, NULL ) ||
|
||||
vips_smartcrop_score( smartcrop, t[0],
|
||||
&left_score ) )
|
||||
return( -1 );
|
||||
|
||||
if( vips_extract_area( in, &t[1],
|
||||
left + width - slice_width, top,
|
||||
slice_width, height, NULL ) ||
|
||||
vips_smartcrop_score( smartcrop, t[1],
|
||||
&right_score ) )
|
||||
return( -1 );
|
||||
|
||||
width -= slice_width;
|
||||
if( left_score < right_score )
|
||||
left += slice_width;
|
||||
}
|
||||
|
||||
if( slice_height > 0 ) {
|
||||
VipsImage **t = (VipsImage **)
|
||||
vips_object_local_array( object, 4 );
|
||||
|
||||
double top_score;
|
||||
double bottom_score;
|
||||
|
||||
if( vips_extract_area( in, &t[0],
|
||||
left, top, width, slice_height, NULL ) ||
|
||||
vips_smartcrop_score( smartcrop, t[0],
|
||||
&top_score ) )
|
||||
return( -1 );
|
||||
|
||||
if( vips_extract_area( in, &t[1],
|
||||
left, top + height - slice_height,
|
||||
width, slice_height, NULL ) ||
|
||||
vips_smartcrop_score( smartcrop, t[1],
|
||||
&bottom_score ) )
|
||||
return( -1 );
|
||||
|
||||
height -= slice_height;
|
||||
if( top_score < bottom_score )
|
||||
top += slice_height;
|
||||
}
|
||||
}
|
||||
if( vips_smartcrop_attention( smartcrop,
|
||||
smartcrop->in, &left, &top ) )
|
||||
return( -1 );
|
||||
break;
|
||||
|
||||
default:
|
||||
@ -285,9 +291,8 @@ vips_smartcrop_build( VipsObject *object )
|
||||
break;
|
||||
}
|
||||
|
||||
/* And our output is the final crop.
|
||||
*/
|
||||
if( vips_extract_area( in, &t[0], left, top, width, height, NULL ) ||
|
||||
if( vips_extract_area( smartcrop->in, &t[0],
|
||||
left, top, smartcrop->width, smartcrop->height, NULL ) ||
|
||||
vips_image_write( t[0], conversion->out ) )
|
||||
return( -1 );
|
||||
|
||||
@ -302,7 +307,6 @@ vips_smartcrop_class_init( VipsSmartcropClass *class )
|
||||
|
||||
VIPS_DEBUG_MSG( "vips_smartcrop_class_init\n" );
|
||||
|
||||
gobject_class->dispose = vips_smartcrop_dispose;
|
||||
gobject_class->set_property = vips_object_set_property;
|
||||
gobject_class->get_property = vips_object_get_property;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user