From 59669be59288a27edfb65fbf4429bcdcc969502a Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Fri, 28 Feb 2020 14:51:21 +0000 Subject: [PATCH 1/2] remove some stray files --- fuzz/common_fuzzer_corpus/x-ascii.ppm | 15 --------------- fuzz/common_fuzzer_corpus/x.csv | 10 ---------- fuzz/common_fuzzer_corpus/x.mat | 11 ----------- fuzz/common_fuzzer_corpus/x.ppm | 6 ------ fuzz/common_fuzzer_corpus/x3.ppm | 6 ------ 5 files changed, 48 deletions(-) delete mode 100644 fuzz/common_fuzzer_corpus/x-ascii.ppm delete mode 100644 fuzz/common_fuzzer_corpus/x.csv delete mode 100644 fuzz/common_fuzzer_corpus/x.mat delete mode 100644 fuzz/common_fuzzer_corpus/x.ppm delete mode 100644 fuzz/common_fuzzer_corpus/x3.ppm diff --git a/fuzz/common_fuzzer_corpus/x-ascii.ppm b/fuzz/common_fuzzer_corpus/x-ascii.ppm deleted file mode 100644 index aa81a14b..00000000 --- a/fuzz/common_fuzzer_corpus/x-ascii.ppm +++ /dev/null @@ -1,15 +0,0 @@ -P2 -#vips2ppm - Fri Aug 23 12:48:07 2019 - -10 10 -255 -96 101 113 118 124 130 136 141 147 150 -81 87 98 101 107 112 117 123 130 135 -73 78 85 90 95 99 103 110 118 124 -46 51 60 67 73 81 87 94 103 109 -34 35 40 48 60 69 77 81 85 88 -28 26 31 36 45 54 59 64 69 72 -32 31 41 39 39 40 45 52 61 66 -38 38 47 42 38 36 38 43 49 53 -37 38 39 39 37 37 37 36 34 32 -36 36 38 36 35 34 35 32 28 25 diff --git a/fuzz/common_fuzzer_corpus/x.csv b/fuzz/common_fuzzer_corpus/x.csv deleted file mode 100644 index 640265ec..00000000 --- a/fuzz/common_fuzzer_corpus/x.csv +++ /dev/null @@ -1,10 +0,0 @@ -96 101 113 118 124 130 136 141 147 150 -81 87 98 101 107 112 117 123 130 135 -73 78 85 90 95 99 103 110 118 124 -46 51 60 67 73 81 87 94 103 109 -34 35 40 48 60 69 77 81 85 88 -28 26 31 36 45 54 59 64 69 72 -32 31 41 39 39 40 45 52 61 66 -38 38 47 42 38 36 38 43 49 53 -37 38 39 39 37 37 37 36 34 32 -36 36 38 36 35 34 35 32 28 25 diff --git a/fuzz/common_fuzzer_corpus/x.mat b/fuzz/common_fuzzer_corpus/x.mat deleted file mode 100644 index bed7279e..00000000 --- a/fuzz/common_fuzzer_corpus/x.mat +++ /dev/null @@ -1,11 +0,0 @@ -10 10 -96 101 113 118 124 130 136 141 147 150 -81 87 98 101 107 112 117 123 130 135 -73 78 85 90 95 99 103 110 118 124 -46 51 60 67 73 81 87 94 103 109 -34 35 40 48 60 69 77 81 85 88 -28 26 31 36 45 54 59 64 69 72 -32 31 41 39 39 40 45 52 61 66 -38 38 47 42 38 36 38 43 49 53 -37 38 39 39 37 37 37 36 34 32 -36 36 38 36 35 34 35 32 28 25 diff --git a/fuzz/common_fuzzer_corpus/x.ppm b/fuzz/common_fuzzer_corpus/x.ppm deleted file mode 100644 index cc0b6072..00000000 --- a/fuzz/common_fuzzer_corpus/x.ppm +++ /dev/null @@ -1,6 +0,0 @@ -P5 -#vips2ppm - Fri Aug 23 12:47:38 2019 - -10 10 -255 -`eqv|‚ˆ“–QWbekpu{‚‡INUZ_cgnv|.3LQDUWL]\SfcZolbzri|.(/)3#.9+8E7FM@QSHYVM`ZQf]Tg' %*$- *6(5=1?C6HG;OL@VOCW,!)!3$+2",0#-1#05(9>/B.""/##0$&/#'.!(.!*. --.+.)-," ,"!/##-!#,&+'-*+*'(#% \ No newline at end of file From ab5d5111230e0a4d3b237c7efba4822d271c0b50 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Fri, 28 Feb 2020 15:22:10 +0000 Subject: [PATCH 2/2] add fuzz corpus --- ChangeLog | 1 + doc/How-it-opens-files.xml | 18 ++-- doc/Using-vipsthumbnail.xml | 83 +++++++----------- fuzz/Makefile.am | 4 +- ...> jpegsave_buffer_fuzzer-5658586599915520} | Bin ...> jpegsave_buffer_fuzzer-5673786296238080} | Bin ...> jpegsave_buffer_fuzzer-5759265708441600} | Bin ... => jpegsave_file_fuzzer-5662041322291200} | Bin ...=> pngsave_buffer_fuzzer-5078454764044288} | Bin ...725568 => sharpen_fuzzer-5203581631725568} | 0 ...639616 => sharpen_fuzzer-5678720198639616} | Bin ...253632 => sharpen_fuzzer-5691855517253632} | Bin ...8896 => smartcrop_fuzzer-5687924892368896} | Bin ...9120 => thumbnail_fuzzer-5676300823429120} | Bin ...7824 => thumbnail_fuzzer-5718717719117824} | Bin ...6768 => thumbnail_fuzzer-5741423734816768} | Bin ...> webpsave_buffer_fuzzer-5207224829345792} | Bin ...> webpsave_buffer_fuzzer-5674696418263040} | Bin 18 files changed, 47 insertions(+), 59 deletions(-) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5658586599915520 => jpegsave_buffer_fuzzer-5658586599915520} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5673786296238080 => jpegsave_buffer_fuzzer-5673786296238080} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5759265708441600 => jpegsave_buffer_fuzzer-5759265708441600} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-jpegsave_file_fuzzer-5662041322291200 => jpegsave_file_fuzzer-5662041322291200} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-pngsave_buffer_fuzzer-5078454764044288 => pngsave_buffer_fuzzer-5078454764044288} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-sharpen_fuzzer-5203581631725568 => sharpen_fuzzer-5203581631725568} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-sharpen_fuzzer-5678720198639616 => sharpen_fuzzer-5678720198639616} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-sharpen_fuzzer-5691855517253632 => sharpen_fuzzer-5691855517253632} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-smartcrop_fuzzer-5687924892368896 => smartcrop_fuzzer-5687924892368896} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-thumbnail_fuzzer-5676300823429120 => thumbnail_fuzzer-5676300823429120} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-thumbnail_fuzzer-5718717719117824 => thumbnail_fuzzer-5718717719117824} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-thumbnail_fuzzer-5741423734816768 => thumbnail_fuzzer-5741423734816768} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-webpsave_buffer_fuzzer-5207224829345792 => webpsave_buffer_fuzzer-5207224829345792} (100%) rename fuzz/common_fuzzer_corpus/{clusterfuzz-testcase-minimized-webpsave_buffer_fuzzer-5674696418263040 => webpsave_buffer_fuzzer-5674696418263040} (100%) diff --git a/ChangeLog b/ChangeLog index b0c34087..9575253c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ - fix a deadlock with --vips-leak [DarthSim] - better gifload behaviour for DISPOSAL_UNSPECIFIED [DarthSim] - ban ppm max_value < 0 +- add fuzz corpus to dist 20/6/19 started 8.9.1 - don't use the new source loaders for new_from_file or new_from_buffer, it diff --git a/doc/How-it-opens-files.xml b/doc/How-it-opens-files.xml index 30388045..c79b13ad 100644 --- a/doc/How-it-opens-files.xml +++ b/doc/How-it-opens-files.xml @@ -16,7 +16,7 @@ This page tries to explain what the different strategies are and when each is used. If you are running into unexpected memory, disc or CPU use, this might be helpful. vips_image_new_from_file() has the official documentation. - +
Direct access This is the fastest and simplest one. The file is mapped directly into the process’s address space and can be read with ordinary pointer access. Small files are completely mapped; large files are mapped in a series of small windows that are shared and which scroll about as pixels are read. Files which are accessed like this can be read by many threads at once, making them especially quick. They also interact well with the computer’s operating system: your OS will use spare memory to cache recently used chunks of the file. @@ -25,10 +25,10 @@ For this to be possible, the file format needs to be a simple dump of a memory array. libvips supports direct access for vips, 8-bit binary ppm/pbm/pnm, analyse and raw. - libvips has a special direct write mode where pixels can be written directly to the file image. This is used for the draw operators. + libvips has a special direct write mode where pixels can be written directly to the file image. This is used for the draw operators. - - +
+
Random access via load library Some image file formats have libraries which allow true random access to image pixels. For example, libtiff lets you read any tile out of a tiled tiff image very quickly. Because the libraries allow true random access, libvips can simply hook the image load library up to the input of the operation pipeline. @@ -39,8 +39,8 @@ libvips can load tiled tiff, tiled OpenEXR, FITS and OpenSlide images in this manner. - - +
+
Full decompression Many image load libraries do not support random access. In order to use images of this type as inputs to pipelines, libvips has to convert them to a random access format first. @@ -54,8 +54,8 @@ This is the slowest and most memory-hungry way to read files, but it’s unavoidable for many file formats. Unless you can use the next one! - - +
+
Sequential access This a fairly recent addition to libvips and is a hybrid of the previous two. @@ -90,7 +90,7 @@ $ vips shrink fred.png jim.png 10 10 This is done automatically in command-line operation. In programs, you need to set access to #VIPS_ACCESS_SEQUENTIAL in calls to functions like vips_image_new_from_file(). - +
diff --git a/doc/Using-vipsthumbnail.xml b/doc/Using-vipsthumbnail.xml index d50b150c..4963eaec 100644 --- a/doc/Using-vipsthumbnail.xml +++ b/doc/Using-vipsthumbnail.xml @@ -21,7 +21,7 @@ $filename = "image.jpg"; $image = Vips\Image::thumbnail($filename, 200, ["height" => 200]); $image->writeToFile("my-thumbnail.jpg"); - +
libvips options vipsthumbnail supports the usual range of vips command-line options. A few of them are useful: @@ -38,8 +38,8 @@ $image->writeToFile("my-thumbnail.jpg"); --vips-info shows a higher level view of the operations that vipsthumbnail is running.  - - +
+
Looping vipsthumbnail can process many images in one command. For example: @@ -56,8 +56,8 @@ $ vipsthumbnail *.jpg $ parallel vipsthumbnail ::: *.jpg - - +
+
Thumbnail size You can set the bounding box of the generated thumbnail with the --size option. For example: @@ -80,8 +80,8 @@ $ vipsthumbnail shark.jpg --size 200x You can append ! to force a resize to the exact target size, breaking the aspect ratio. - - +
+
Cropping vipsthumbnail normally shrinks images to fit within the box set by --size. You can use the --smartcrop option to crop to fill the box instead. Excess pixels are trimmed away using the strategy you set. For example: @@ -92,30 +92,28 @@ $ vipsthumbnail owl.jpg --smartcrop attention -s 128 Where owl.jpg is an off-centre composition: -
- + + - - -
+ + Gives this result: -
- + + - - -
+ + First it shrinks the image to get the vertical axis to 128 pixels, then crops down to 128 pixels across using the attention strategy. This one searches the image for features which might catch a human eye, see vips_smartcrop() for details. - - +
+
Linear light Shrinking images involves combining many pixels into one. Arithmetic averaging really ought to be in terms of the number of photons, but (for historical reasons) the values stored in image files are usually related to the voltage that should be applied to the electron gun in a CRT display. @@ -142,8 +140,8 @@ real 0m4.660s user 0m4.640s sys 0m0.016s - - +
+
Output directory You set the thumbnail write parameters with the -o option. This is a pattern which the input filename is pasted into to produce the output filename. For example: @@ -172,8 +170,8 @@ $ vipsthumbnail fred.jpg ../jim.tif -o mythumbs/tn_%s.jpg Now both input files will have thumbnails written to a subdirectory of their current directory. - - +
+
Output format and options You can use -o to specify the thumbnail image format too. For example:  @@ -247,8 +245,8 @@ $ vipsthumbnail 42-32157534.jpg -o x.jpg[optimize_coding,strip] $ ls -l x.jpg -rw-r–r– 1 john john 3600 Nov 12 21:27 x.jpg - - +
+
Colour management vipsthumbnail will optionally put images through LittleCMS for you. You can use this to move all thumbnails to the same colour space. All web browsers assume that images without an ICC profile are in sRGB colourspace, so if you move your thumbnails to sRGB, you can strip all the embedded profiles. This can save several kb per thumbnail. @@ -265,12 +263,15 @@ $ ls -l tn_shark.jpg Now encode with sRGB and delete any embedded profile: -$ vipsthumbnail shark.jpg --eprofile /usr/share/color/icc/sRGB.icc --delete +$ vipsthumbnail shark.jpg --eprofile srgb --delete $ ls -l tn_shark.jpg -rw-r–r– 1 john john 4229 Nov  9 14:33 tn_shark.jpg - It’ll look identical to a user, but be almost half the size. + (You can use the filename of any RGB profile. The magic string srgb selects a high-quality sRGB profile that’s built into libvips.) + + + tn_shark.jpg will look identical to a user, but it’s almost half the size. You can also specify a fallback input profile to use if the image has no embedded one. This can often happen with CMYK images, producing an error message like: @@ -284,28 +285,13 @@ vips_colourspace: no known route from 'cmyk' to 'srgb' If you supply a CMYK profile, it will be able to convert the image, for example: -$ vipsthumbnail kgdev.jpg --iprofile /usr/share/color/icc/colord/FOGRA28L_webcoated.icc +$ vipsthumbnail kgdev.jpg --iprofile cmyk - I’ve had good results with this profile: + (As before, the magic string cmyk selects a high-quality CMYK profile that’s built into libvips, but you can use any CMYK profile you like.) - - https://github.com/libvips/nip2/blob/master/share/nip2/data/cmyk.icm - - - It makes nice-looking images from most CMYK files, and is completely free. - - - - Auto-rotate - - Many JPEG files have a hint set in the header giving the image orientation. If you strip out the metadata, this hint will be lost, and the image will appear to be rotated. - - - If you use the --rotate option, vipsthumbnail examines the image header and if there’s an orientation tag, applies and removes it. - - - +
+
Final suggestion Putting all this together, I suggest this as a sensible set of options: @@ -314,10 +300,9 @@ $ vipsthumbnail kgdev.jpg --iprofile /usr/share/color/icc/colord/FOGRA28L_webcoa $ vipsthumbnail fred.jpg \ --size 128 \ -o tn_%s.jpg[optimize_coding,strip] \ - --eprofile /usr/share/color/icc/sRGB.icc \ - --rotate + --eprofile srgb - +
diff --git a/fuzz/Makefile.am b/fuzz/Makefile.am index edee8cbd..9effbde2 100644 --- a/fuzz/Makefile.am +++ b/fuzz/Makefile.am @@ -25,4 +25,6 @@ libstandaloneengine_a_SOURCES = StandaloneFuzzTargetMain.c check_PROGRAMS = $(FUZZPROGS) noinst_LIBRARIES = $(FUZZLIBS) -EXTRA_DIST = $(TESTS) +EXTRA_DIST = \ + $(TESTS) \ + common_fuzzer_corpus diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5658586599915520 b/fuzz/common_fuzzer_corpus/jpegsave_buffer_fuzzer-5658586599915520 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5658586599915520 rename to fuzz/common_fuzzer_corpus/jpegsave_buffer_fuzzer-5658586599915520 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5673786296238080 b/fuzz/common_fuzzer_corpus/jpegsave_buffer_fuzzer-5673786296238080 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5673786296238080 rename to fuzz/common_fuzzer_corpus/jpegsave_buffer_fuzzer-5673786296238080 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5759265708441600 b/fuzz/common_fuzzer_corpus/jpegsave_buffer_fuzzer-5759265708441600 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_buffer_fuzzer-5759265708441600 rename to fuzz/common_fuzzer_corpus/jpegsave_buffer_fuzzer-5759265708441600 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_file_fuzzer-5662041322291200 b/fuzz/common_fuzzer_corpus/jpegsave_file_fuzzer-5662041322291200 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-jpegsave_file_fuzzer-5662041322291200 rename to fuzz/common_fuzzer_corpus/jpegsave_file_fuzzer-5662041322291200 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-pngsave_buffer_fuzzer-5078454764044288 b/fuzz/common_fuzzer_corpus/pngsave_buffer_fuzzer-5078454764044288 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-pngsave_buffer_fuzzer-5078454764044288 rename to fuzz/common_fuzzer_corpus/pngsave_buffer_fuzzer-5078454764044288 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-sharpen_fuzzer-5203581631725568 b/fuzz/common_fuzzer_corpus/sharpen_fuzzer-5203581631725568 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-sharpen_fuzzer-5203581631725568 rename to fuzz/common_fuzzer_corpus/sharpen_fuzzer-5203581631725568 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-sharpen_fuzzer-5678720198639616 b/fuzz/common_fuzzer_corpus/sharpen_fuzzer-5678720198639616 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-sharpen_fuzzer-5678720198639616 rename to fuzz/common_fuzzer_corpus/sharpen_fuzzer-5678720198639616 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-sharpen_fuzzer-5691855517253632 b/fuzz/common_fuzzer_corpus/sharpen_fuzzer-5691855517253632 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-sharpen_fuzzer-5691855517253632 rename to fuzz/common_fuzzer_corpus/sharpen_fuzzer-5691855517253632 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-smartcrop_fuzzer-5687924892368896 b/fuzz/common_fuzzer_corpus/smartcrop_fuzzer-5687924892368896 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-smartcrop_fuzzer-5687924892368896 rename to fuzz/common_fuzzer_corpus/smartcrop_fuzzer-5687924892368896 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-thumbnail_fuzzer-5676300823429120 b/fuzz/common_fuzzer_corpus/thumbnail_fuzzer-5676300823429120 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-thumbnail_fuzzer-5676300823429120 rename to fuzz/common_fuzzer_corpus/thumbnail_fuzzer-5676300823429120 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-thumbnail_fuzzer-5718717719117824 b/fuzz/common_fuzzer_corpus/thumbnail_fuzzer-5718717719117824 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-thumbnail_fuzzer-5718717719117824 rename to fuzz/common_fuzzer_corpus/thumbnail_fuzzer-5718717719117824 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-thumbnail_fuzzer-5741423734816768 b/fuzz/common_fuzzer_corpus/thumbnail_fuzzer-5741423734816768 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-thumbnail_fuzzer-5741423734816768 rename to fuzz/common_fuzzer_corpus/thumbnail_fuzzer-5741423734816768 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-webpsave_buffer_fuzzer-5207224829345792 b/fuzz/common_fuzzer_corpus/webpsave_buffer_fuzzer-5207224829345792 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-webpsave_buffer_fuzzer-5207224829345792 rename to fuzz/common_fuzzer_corpus/webpsave_buffer_fuzzer-5207224829345792 diff --git a/fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-webpsave_buffer_fuzzer-5674696418263040 b/fuzz/common_fuzzer_corpus/webpsave_buffer_fuzzer-5674696418263040 similarity index 100% rename from fuzz/common_fuzzer_corpus/clusterfuzz-testcase-minimized-webpsave_buffer_fuzzer-5674696418263040 rename to fuzz/common_fuzzer_corpus/webpsave_buffer_fuzzer-5674696418263040