From 38e0cbb12a784c34967cbf31684eec833ca93d80 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Fri, 15 Feb 2019 17:27:20 +0000 Subject: [PATCH] speed up tilecache with a simple LRU queue of recent tiles better cmap handling for gifload as well --- libvips/conversion/tilecache.c | 93 +++++++++++----------------------- libvips/foreign/gifload.c | 67 ++++++++++++++---------- 2 files changed, 71 insertions(+), 89 deletions(-) diff --git a/libvips/conversion/tilecache.c b/libvips/conversion/tilecache.c index 13ba6553..c666c580 100644 --- a/libvips/conversion/tilecache.c +++ b/libvips/conversion/tilecache.c @@ -35,6 +35,8 @@ * - terminate on tile calc error * 7/3/17 * - remove "access" on linecache, use the base class instead + * 15/2/19 + * - remove the search for LRU, have a gqueue instead, much faster */ /* @@ -115,8 +117,6 @@ typedef struct _VipsTile { * pointer is NULL. */ VipsRect pos; - - int time; /* Time of last use for flush */ } VipsTile; typedef struct _VipsBlockCache { @@ -126,15 +126,20 @@ typedef struct _VipsBlockCache { int tile_width; int tile_height; int max_tiles; + + /* access doesn't actually have any effect now. It used to set + * the order for the recycle queue, but there's no efficient way to do + * top-most, so we're just always LRU. + */ VipsAccess access; gboolean threaded; gboolean persistent; - int time; /* Update ticks for LRU here */ int ntiles; /* Current cache size */ GMutex *lock; /* Lock everything here */ GCond *new_tile; /* A new tile is ready */ GHashTable *tiles; /* Tiles, hashed by coordinates */ + GQueue *recycle; /* Queue of unreffed tiles to reuse */ } VipsBlockCache; typedef VipsConversionClass VipsBlockCacheClass; @@ -166,18 +171,11 @@ vips_block_cache_dispose( GObject *gobject ) if( cache->tiles ) g_assert( g_hash_table_size( cache->tiles ) == 0 ); VIPS_FREEF( g_hash_table_destroy, cache->tiles ); + VIPS_FREEF( g_queue_free, cache->recycle ); G_OBJECT_CLASS( vips_block_cache_parent_class )->dispose( gobject ); } -static void -vips_tile_touch( VipsTile *tile ) -{ - g_assert( tile->cache->ntiles >= 0 ); - - tile->time = tile->cache->time++; -} - static int vips_tile_move( VipsTile *tile, int x, int y ) { @@ -215,7 +213,6 @@ vips_tile_new( VipsBlockCache *cache, int x, int y ) tile->state = VIPS_TILE_STATE_PEND; tile->ref_count = 0; tile->region = NULL; - tile->time = cache->time; tile->pos.left = x; tile->pos.top = y; tile->pos.width = cache->tile_width; @@ -256,45 +253,6 @@ vips_tile_search( VipsBlockCache *cache, int x, int y ) return( tile ); } -typedef struct _VipsTileSearch { - VipsTile *tile; - - int oldest; - int topmost; -} VipsTileSearch; - -static void -vips_tile_search_recycle( gpointer key, gpointer value, gpointer user_data ) -{ - VipsTile *tile = (VipsTile *) value; - VipsBlockCache *cache = tile->cache; - VipsTileSearch *search = (VipsTileSearch *) user_data; - - /* Only consider unreffed tiles for recycling. - */ - if( !tile->ref_count ) { - switch( cache->access ) { - case VIPS_ACCESS_RANDOM: - if( tile->time < search->oldest ) { - search->oldest = tile->time; - search->tile = tile; - } - break; - - case VIPS_ACCESS_SEQUENTIAL: - case VIPS_ACCESS_SEQUENTIAL_UNBUFFERED: - if( tile->pos.top < search->topmost ) { - search->topmost = tile->pos.top; - search->tile = tile; - } - break; - - default: - g_assert_not_reached(); - } - } -} - /* Find existing tile, make a new tile, or if we have a full set of tiles, * reuse one. */ @@ -302,7 +260,6 @@ static VipsTile * vips_tile_find( VipsBlockCache *cache, int x, int y ) { VipsTile *tile; - VipsTileSearch search; /* In cache already? */ @@ -324,13 +281,12 @@ vips_tile_find( VipsBlockCache *cache, int x, int y ) return( tile ); } - /* Reuse an old one. + /* Reuse an old one, if there are any. */ - search.oldest = cache->time; - search.topmost = cache->in->Ysize; - search.tile = NULL; - g_hash_table_foreach( cache->tiles, vips_tile_search_recycle, &search ); - tile = search.tile; + if( cache->recycle ) + /* Gets removed from the recycle list on _ref. + */ + tile = g_queue_peek_head( cache->recycle ); if( !tile ) { /* There are no tiles we can reuse -- we have to make another @@ -497,7 +453,6 @@ vips_block_cache_init( VipsBlockCache *cache ) cache->threaded = FALSE; cache->persistent = FALSE; - cache->time = 0; cache->ntiles = 0; cache->lock = vips_g_mutex_new(); cache->new_tile = vips_g_cond_new(); @@ -506,6 +461,7 @@ vips_block_cache_init( VipsBlockCache *cache ) (GEqualFunc) vips_rect_equal, NULL, (GDestroyNotify) vips_tile_destroy ); + cache->recycle = g_queue_new(); } typedef struct _VipsTileCache { @@ -523,6 +479,15 @@ vips_tile_unref( VipsTile *tile ) g_assert( tile->ref_count > 0 ); tile->ref_count -= 1; + + if( tile->ref_count == 0 ) { + /* Place at the end of the recycle queue. We pop from the + * front when selecting an unused tile for reuse. + */ + g_assert( !g_queue_find( tile->cache->recycle, tile ) ); + + g_queue_push_tail( tile->cache->recycle, tile ); + } } static void @@ -531,6 +496,12 @@ vips_tile_ref( VipsTile *tile ) tile->ref_count += 1; g_assert( tile->ref_count > 0 ); + + if( tile->ref_count == 1 ) { + g_assert( g_queue_find( tile->cache->recycle, tile ) ); + + g_queue_remove( tile->cache->recycle, tile ); + } } static void @@ -571,8 +542,6 @@ vips_tile_cache_ref( VipsBlockCache *cache, VipsRect *r ) return( NULL ); } - vips_tile_touch( tile ); - vips_tile_ref( tile ); /* We must append, since we want to keep tile ordering @@ -713,8 +682,6 @@ vips_tile_cache_gen( VipsRegion *or, tile->state = VIPS_TILE_STATE_DATA; - vips_tile_touch( tile ); - /* Let everyone know there's a new DATA tile. * They need to all check their work lists. */ diff --git a/libvips/foreign/gifload.c b/libvips/foreign/gifload.c index 339f68c8..a4380ec4 100644 --- a/libvips/foreign/gifload.c +++ b/libvips/foreign/gifload.c @@ -169,12 +169,14 @@ typedef struct _VipsForeignLoadGif { int dispose; /* Set for EOF detected. - * - * FIXME ... do we need this? - * */ gboolean eof; + /* The current cmap unpacked to a simple LUT. Each uint32 is really an + * RGBA pixel ready to be blasted into @frame. + */ + guint32 cmap[256]; + /* As we scan the file, the index of the transparent pixel for this * frame. */ @@ -657,26 +659,24 @@ vips_foreign_load_gif_header( VipsForeignLoad *load ) } static void -vips_foreign_load_gif_render_line( VipsForeignLoadGif *gif, - int width, VipsPel * restrict q, VipsPel * restrict p ) +vips_foreign_load_gif_build_cmap( VipsForeignLoadGif *gif ) { ColorMapObject *map = gif->file->Image.ColorMap ? gif->file->Image.ColorMap : gif->file->SColorMap; - int x; + int v; + + for( v = 0; v < 256; v++ ) { + VipsPel *q = (VipsPel *) &gif->cmap[v]; - for( x = 0; x < width; x++ ) { - VipsPel v = p[x]; - if( map && - v < map->ColorCount && - v != gif->transparency ) { + v < map->ColorCount ) { q[0] = map->Colors[v].Red; q[1] = map->Colors[v].Green; q[2] = map->Colors[v].Blue; q[3] = 255; } - else if( v != gif->transparency ) { + else { /* If there's no map, just save the index. */ q[0] = v; @@ -684,21 +684,32 @@ vips_foreign_load_gif_render_line( VipsForeignLoadGif *gif, q[2] = v; q[3] = 255; } - else if( gif->dispose == DISPOSE_DO_NOT ) { - /* Transparent pixels let the previous frame show - * through, ie., do nothing. - */ - } - else { - /* All other modes are just transparent. - */ - q[0] = 0; - q[1] = 0; - q[2] = 0; - q[3] = 0; - } + } +} - q += 4; +static void +vips_foreign_load_gif_render_line( VipsForeignLoadGif *gif, + int width, VipsPel * restrict q, VipsPel * restrict p ) +{ + guint32 *iq; + int x; + + iq = (guint32 *) q; + for( x = 0; x < width; x++ ) { + VipsPel v = p[x]; + + if( v == gif->transparency ) { + /* In DISPOSE_DO_NOT mode, the previous frame shows + * through (ie. we do nothing). In all other modes, + * it's just transparent. + */ + if( gif->dispose != DISPOSE_DO_NOT ) + iq[x] = 0; + } + else + /* Blast in the RGBA for this value. + */ + iq[x] = gif->cmap[v]; } } @@ -710,6 +721,10 @@ vips_foreign_load_gif_render( VipsForeignLoadGif *gif ) { GifFileType *file = gif->file; + /* Update the colour map for this frame. + */ + vips_foreign_load_gif_build_cmap( gif ); + if( file->Image.Interlace ) { int i;