From 801111a2fa98690cfb41106ab890a79e307b713a Mon Sep 17 00:00:00 2001
From: John Cupitt <jcupitt@gmail.com>
Date: Tue, 8 Sep 2020 13:50:14 +0100
Subject: [PATCH] better dint rules

We had some special cases coded for dhint inheritance, but they could
fail in some edge cases. Revert to something simpler and more
predictable.

see https://github.com/libvips/libvips/issues/1810
---
 ChangeLog                  |  1 +
 libvips/iofuncs/generate.c | 29 +++++++++++------------------
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f5181adb..eb41e6d2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,6 @@
 6/9/20 started 8.10.2
 - update magicksave/load profile handling [kelilevi]
+- better demand hint rules [kaas3000]
 
 9/8/20 started 8.10.1
 - fix markdown -> xml conversion in doc generation
diff --git a/libvips/iofuncs/generate.c b/libvips/iofuncs/generate.c
index 8e8e53a4..575c0f98 100644
--- a/libvips/iofuncs/generate.c
+++ b/libvips/iofuncs/generate.c
@@ -297,25 +297,18 @@ vips__demand_hint_array( VipsImage *image,
 		if( in[i]->dhint == VIPS_DEMAND_STYLE_ANY )
 			nany++;
 
+	/* Find the most restrictive of all the hints available to us.
+	 *
+	 * We have tried to be smarter about this in the past -- for example,
+	 * detecting all ANY inputs and ignoring the hint in this case, but
+	 * there are inevitably odd cases which cause problems. For example,
+	 * new_from_memory, resize, affine, write_to_memory would run with
+	 * FATSTRIP.
+	 */
 	set_hint = hint;
-	if( len == 0 ) 
-		/* No input images? Just set the requested hint. We don't 
-		 * force ANY, since the operation might be something like 
-		 * tiled read of an EXR image, where we certainly don't want 
-		 * ANY.
-		 */
-		;
-	else if( nany == len ) 
-		/* Special case: if all the inputs are ANY, then output can 
-		 * be ANY regardless of what this function wants. 
-		 */
-		set_hint = VIPS_DEMAND_STYLE_ANY;
-	else
-		/* Find the most restrictive of all the hints available to us.
-		 */
-		for( i = 0; i < len; i++ )
-			set_hint = (VipsDemandStyle) VIPS_MIN( 
-				(int) set_hint, (int) in[i]->dhint );
+	for( i = 0; i < len; i++ )
+		set_hint = (VipsDemandStyle) VIPS_MIN( 
+			(int) set_hint, (int) in[i]->dhint );
 
 	image->dhint = set_hint;