Merge branch 'master' into loader-minimise-experiment

.gitignore

@@ -27,6 +27,8 @@ Makefile.in
 TAGS
 tags
 *.o
+*.a
+fuzz/*_fuzzer
 Vips-8.0.gir
 Vips-8.0.typelib
 .*.swp

.travis.yml

@@ -10,9 +10,10 @@ before_script:
     --with-jpeg-libraries=$JPEG/lib
   - make -j$JOBS -s 
 script:
-  - make -Ctest -j$JOBS -s V=0 VERBOSE=1 check
+  - make -j$JOBS -s -k V=0 VERBOSE=1 check
   - LD_LIBRARY_PATH=$PWD/libvips/.libs
     DYLD_LIBRARY_PATH=$PWD/libvips/.libs
+    LD_PRELOAD=$ASAN_DSO
     $PYTHON -m pytest -v test/test-suite
 
 matrix:
@@ -23,6 +24,7 @@ matrix:
     - os: linux
       sudo: required
       dist: xenial
+      compiler: gcc
       env:
         - PYTHON=python2
         - PYVIPS_VERSION=master
@@ -30,6 +32,29 @@ matrix:
         - JOBS=`nproc`
       cache: ccache
 
+    - os: linux
+      sudo: required
+      dist: xenial
+      compiler: clang
+      env:
+        - PYTHON=python2
+        - PYVIPS_VERSION=master
+        - JPEG=/usr
+        - JOBS=`nproc`
+        - CFLAGS="-fsanitize=address -fno-omit-frame-pointer -fopenmp"
+        - LDFLAGS="-fsanitize=address -dynamic-asan -fopenmp=libiomp5"
+        - ASAN_DSO=/usr/local/clang-7.0.0/lib/clang/7.0.0/lib/linux/libclang_rt.asan-x86_64.so
+        - LSAN_OPTIONS="suppressions=$TRAVIS_BUILD_DIR/lsan.supp"
+        - FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1
+      install:
+        # add support for WebP
+        - wget http://archive.ubuntu.com/ubuntu/pool/main/libw/libwebp/libwebp-dev_0.6.1-2_amd64.deb
+        - wget http://archive.ubuntu.com/ubuntu/pool/main/libw/libwebp/libwebpdemux2_0.6.1-2_amd64.deb
+        - wget http://archive.ubuntu.com/ubuntu/pool/main/libw/libwebp/libwebpmux3_0.6.1-2_amd64.deb
+        - wget http://archive.ubuntu.com/ubuntu/pool/main/libw/libwebp/libwebp6_0.6.1-2_amd64.deb
+        - sudo dpkg -i *.deb
+      cache: ccache
+
     - os: osx
       osx_image: xcode10.1
       env:
@@ -79,6 +104,7 @@ addons:
       - libgsf-1-dev 
       - libopenslide-dev
       - libffi-dev
+      - libiomp-dev
   homebrew:
     packages:
       - ccache

ChangeLog

@@ -7,6 +7,7 @@
 - add "compression" option to heifsave [lovell]
 - support webp and zstd compression in tiff
 - use "minimise" to close input early
+- integrate support for oss-fuzz [omira-sch]
 
 9/7/19 started 8.8.2
 - better early shutdown in readers

Makefile.am

@@ -5,7 +5,8 @@ SUBDIRS = \
 	po \
 	man \
 	doc \
-	test 
+	test \
+	fuzz
 
 EXTRA_DIST = \
 	m4 \
@@ -14,6 +15,7 @@ EXTRA_DIST = \
 	vips.pc.in \
 	vips-cpp.pc.in \
 	libvips.supp \
+	lsan.supp \
 	depcomp \
 	README.md 
 

configure.ac

@@ -1388,6 +1388,13 @@ if test x"$with_libexif" != x"no"; then
   CPPFLAGS="$save_CPPFLAGS"
 fi
 
+# fuzzing
+AC_ARG_VAR([LIB_FUZZING_ENGINE],
+           [fuzzing library, e.g. /path/to/libFuzzer.a])
+if test x"$LIB_FUZZING_ENGINE" = x; then
+  LIB_FUZZING_ENGINE="libstandaloneengine.a"
+fi
+
 # Gather all up for VIPS_CFLAGS, VIPS_INCLUDES, VIPS_LIBS 
 # sort includes to get longer, more specific dirs first
 # helps, for example, selecting graphicsmagick over imagemagick
@@ -1452,6 +1459,7 @@ AC_OUTPUT([
 	doc/Makefile
 	doc/libvips-docs.xml
 	po/Makefile.in
+	fuzz/Makefile
 ])
 
 AC_MSG_RESULT([dnl

fuzz/Makefile.am

@@ -0,0 +1,25 @@
+TESTS = \
+	test_fuzz.sh
+
+FUZZPROGS = \
+	jpegsave_buffer_fuzzer \
+	pngsave_buffer_fuzzer \
+	webpsave_buffer_fuzzer \
+	sharpen_fuzzer \
+	thumbnail_fuzzer
+
+AM_DEFAULT_SOURCE_EXT = .cc
+
+FUZZLIBS = libstandaloneengine.a
+
+# Include debug symbols by default as recommended by libfuzzer
+AM_CXXFLAGS = -g -I${top_srcdir}/libvips/include @VIPS_CFLAGS@ @VIPS_INCLUDES@
+AM_LDFLAGS = @LDFLAGS@
+LDADD = ${top_builddir}/libvips/libvips.la @VIPS_LIBS@ @LIB_FUZZING_ENGINE@
+
+libstandaloneengine_a_SOURCES = StandaloneFuzzTargetMain.c
+
+check_PROGRAMS = $(FUZZPROGS)
+noinst_LIBRARIES = $(FUZZLIBS)
+
+EXTRA_DIST = $(TESTS)

fuzz/StandaloneFuzzTargetMain.c

@@ -0,0 +1,46 @@
+/*===- StandaloneFuzzTargetMain.c - standalone main() for fuzz targets. ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This main() function can be linked to a fuzz target (i.e. a library
+// that exports LLVMFuzzerTestOneInput() and possibly LLVMFuzzerInitialize())
+// instead of libFuzzer. This main() function will not perform any fuzzing
+// but will simply feed all input files one by one to the fuzz target.
+//
+// Use this file to provide reproducers for bugs when linking against libFuzzer
+// or other fuzzing engine is undesirable.
+//===----------------------------------------------------------------------===*/
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+extern int LLVMFuzzerTestOneInput(const unsigned char *data, size_t size);
+extern int LLVMFuzzerInitialize(int *argc, char ***argv);
+int main(int argc, char **argv) {
+  const char *progname;
+  if ((progname = strrchr(argv[0], '/')))
+    progname++;
+  else
+    progname = argv[0];
+  fprintf(stderr, "%s: running %d inputs\n", progname, argc - 1);
+  LLVMFuzzerInitialize(&argc, &argv);
+  for (int i = 1; i < argc; i++) {
+    fprintf(stderr, "Running: %s\n", argv[i]);
+    FILE *f = fopen(argv[i], "r+");
+    assert(f);
+    fseek(f, 0, SEEK_END);
+    long len = ftell(f);
+    fseek(f, 0, SEEK_SET);
+    unsigned char *buf = (unsigned char*)malloc(len);
+    size_t n_read = fread(buf, 1, len, f);
+    fclose(f);
+    assert(n_read == len);
+    LLVMFuzzerTestOneInput(buf, len);
+    free(buf);
+    fprintf(stderr, "Done:    %s: (%zd bytes)\n", argv[i], n_read);
+  }
+}

fuzz/jpegsave_buffer_fuzzer.cc

@@ -0,0 +1,30 @@
+#include <vips/vips.h>
+
+extern "C" int
+LLVMFuzzerInitialize( int *argc, char ***argv )
+{
+	vips_concurrency_set( 1 );
+	return( 0 );
+}
+
+extern "C" int
+LLVMFuzzerTestOneInput( const guint8 *data, size_t size )
+{
+	VipsImage *image;
+	size_t len;
+	void *buf;
+
+	if( !(image = vips_image_new_from_buffer( data, size, "", NULL )) ) {
+		return( 0 );
+	}
+
+	if( vips_jpegsave_buffer( image, &buf, &len, NULL ) ) {
+		g_object_unref( image );
+		return( 0 );
+	}
+
+	g_free( buf );
+	g_object_unref( image );
+
+	return( 0 );
+}

fuzz/pngsave_buffer_fuzzer.cc

@@ -0,0 +1,30 @@
+#include <vips/vips.h>
+
+extern "C" int
+LLVMFuzzerInitialize( int *argc, char ***argv )
+{
+	vips_concurrency_set( 1 );
+	return( 0 );
+}
+
+extern "C" int
+LLVMFuzzerTestOneInput( const guint8 *data, size_t size )
+{
+	VipsImage *image;
+	void *buf;
+	size_t len;
+
+	if( !(image = vips_image_new_from_buffer( data, size, "", NULL )) ) {
+		return( 0 );
+	}
+
+	if( vips_pngsave_buffer( image, &buf, &len, NULL ) ) {
+		g_object_unref( image );
+		return( 0 );
+	}
+
+	g_free( buf );
+	g_object_unref( image );
+
+	return( 0 );
+}

fuzz/sharpen_fuzzer.cc

@@ -0,0 +1,31 @@
+#include <vips/vips.h>
+
+extern "C" int
+LLVMFuzzerInitialize( int *argc, char ***argv )
+{
+	vips_concurrency_set( 1 );
+	return( 0 );
+}
+
+extern "C" int
+LLVMFuzzerTestOneInput( const guint8 *data, size_t size )
+{
+	VipsImage *in, *out;
+	double d;
+
+	if( !(in = vips_image_new_from_buffer( data, size, "", NULL )) ) {
+		return( 0 );
+	}
+
+	if( vips_sharpen( in, &out, NULL ) ) {
+		g_object_unref( in );
+		return( 0 );
+	}
+
+	vips_avg( out, &d, NULL );
+
+	g_object_unref( out );
+	g_object_unref( in );
+
+	return( 0 );
+}

fuzz/test_fuzz.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+#set -x
+set -e
+
+# Glib is build without -fno-omit-frame-pointer. We need
+# to disable the fast unwinder to get full stacktraces.
+export ASAN_OPTIONS="fast_unwind_on_malloc=0:allocator_may_return_null=1"
+
+# Hide all warning messages from vips.
+export VIPS_WARNING=0
+
+ret=0
+
+for fuzzer in *_fuzzer; do
+	find "${fuzzer}_corpus" -type f -not -empty -print0 \
+	  | xargs -0 -n1 "./$fuzzer" || ret=1
+done
+
+exit $ret

fuzz/thumbnail_fuzzer.cc

@@ -0,0 +1,42 @@
+#include <vips/vips.h>
+
+extern "C" int
+LLVMFuzzerInitialize( int *argc, char ***argv )
+{
+	vips_concurrency_set( 1 );
+	return( 0 );
+}
+
+extern "C" int
+LLVMFuzzerTestOneInput( const guint8 *data, size_t size )
+{
+	VipsImage *in, *out;
+	size_t width, height;
+	double d;
+
+	if( !(in = vips_image_new_from_buffer( data, size, "", NULL )) ) {
+		return( 0 );
+	}
+
+	width = in->Xsize;
+	height = in->Ysize;
+
+	/* Skip big images. It is likely to timeout.
+	 */
+	if ( width * height > 256 * 256 ) {
+		g_object_unref( in );
+		return( 0 );
+	}
+
+	if( vips_thumbnail_image( in, &out, 42, NULL ) ) {
+		g_object_unref( in );
+		return( 0 );
+	}
+
+	vips_avg( out, &d, NULL );
+
+	g_object_unref( out );
+	g_object_unref( in );
+
+	return( 0 );
+}

fuzz/webpsave_buffer_fuzzer.cc

@@ -0,0 +1,30 @@
+#include <vips/vips.h>
+
+extern "C" int
+LLVMFuzzerInitialize( int *argc, char ***argv )
+{
+	vips_concurrency_set( 1 );
+	return( 0 );
+}
+
+extern "C" int
+LLVMFuzzerTestOneInput( const guint8 *data, size_t size )
+{
+	VipsImage *image;
+	void *buf;
+	size_t len;
+
+	if( !(image = vips_image_new_from_buffer( data, size, "", NULL )) ) {
+		return( 0 );
+	}
+
+	if( vips_webpsave_buffer( image, &buf, &len, NULL ) ) {
+		g_object_unref( image );
+		return( 0 );
+	}
+
+	g_free( buf );
+	g_object_unref( image );
+
+	return( 0 );
+}

libvips/foreign/vipspng.c

@@ -249,6 +249,13 @@ read_new( VipsImage *out, gboolean fail )
 		PNG_SKIP_sRGB_CHECK_PROFILE, PNG_OPTION_ON );
 #endif /*PNG_SKIP_sRGB_CHECK_PROFILE*/
 
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+	/* Disable CRC checking in fuzzing mode.
+	 */
+	png_set_crc_action( read->pPng,
+		PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE );
+#endif /*FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION*/
+
 	/* Catch PNG errors from png_create_info_struct().
 	 */
 	if( setjmp( png_jmpbuf( read->pPng ) ) ) 

libvips/fuzz/Makefile.am

@@ -1,6 +0,0 @@
-noinst_LTLIBRARIES = libfuzz.la
-
-libfuzz_la_SOURCES = \
-	fuzz_new_from_buffer.c
-
-AM_CPPFLAGS = -I${top_srcdir}/libvips/include @VIPS_CFLAGS@ @VIPS_INCLUDES@ 

libvips/fuzz/fuzz_new_from_buffer.c

@@ -1,64 +0,0 @@
-/* fuzz targets for libfuzzer
- * 
- * 28/7/17
- * 	- first attempt
- */
-
-/*
-
-    This file is part of VIPS.
-    
-    VIPS is free software; you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-    02110-1301  USA
-
- */
-
-/*
-
-    These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk
-
- */
-
-/*
-#define VIPS_DEBUG
- */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif /*HAVE_CONFIG_H*/
-#include <vips/intl.h>
-
-#include <vips/vips.h>
-
-int 
-vips__fuzztarget_new_from_buffer( const guint8 *data, size_t size ) 
-{
-	VipsImage *image;
-	double d;
-
-	/* Have one for each format as well.
-	 */
-	if( !(image = vips_image_new_from_buffer( data, size, "", NULL )) )
-		/* libfuzzer does not allow error return.
-		 */
-		return( 0 );
-
-	if( vips_avg( image, &d, NULL ) )
-		return( 0 );
-
-	g_object_unref( image ); 
-
-	return( 0 );
-}

lsan.supp

@@ -0,0 +1,5 @@
+leak:python2.7
+leak:libfontconfig.so
+leak:libglib-2.0.so
+leak:libIlmImf-2_2.so
+leak:libstdc++.so