Multisite: More specific caching for get_dirsize.

Instead of one cache entry for all upload folders for a site on multisite, this now caches for each folder and invalidates that cache based on context. In multisite, this should speed up `get_dirsize` calls since older directories that are much less likely to change will no longer have the size recalculated.

Props janthiel, A5hleyRich, batmoo.
Fixes #19879.


git-svn-id: https://develop.svn.wordpress.org/trunk@49212 602fd350-edb4-49c9-b593-d223f7449a82

src/wp-admin/includes/file.php

@@ -928,7 +928,7 @@ function _wp_handle_upload( &$file, $overrides, $time, $action ) {
 	$url = $uploads['url'] . "/$filename";
 
 	if ( is_multisite() ) {
-		delete_transient( 'dirsize_cache' );
+		invalidate_dirsize_cache( $new_file );
 	}
 
 	/**

src/wp-includes/functions.php

@@ -2740,6 +2740,10 @@ function wp_upload_bits( $name, $deprecated, $bits, $time = null ) {
 	// Compute the URL.
 	$url = $upload['url'] . "/$filename";
 
+	if ( is_multisite() ) {
+		invalidate_dirsize_cache( $new_file );
+	}
+
 	/** This filter is documented in wp-admin/includes/file.php */
 	return apply_filters(
 		'wp_handle_upload',
@@ -7560,26 +7564,16 @@ function wp_direct_php_update_button() {
  * @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout.
  */
 function get_dirsize( $directory, $max_execution_time = null ) {
-	$dirsize = get_transient( 'dirsize_cache' );
-
-	if ( is_array( $dirsize ) && isset( $dirsize[ $directory ]['size'] ) ) {
-		return $dirsize[ $directory ]['size'];
-	}
-
-	if ( ! is_array( $dirsize ) ) {
-		$dirsize = array();
-	}
 
 	// Exclude individual site directories from the total when checking the main site of a network,
 	// as they are subdirectories and should not be counted.
 	if ( is_multisite() && is_main_site() ) {
-		$dirsize[ $directory ]['size'] = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time );
+		$size = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time );
 	} else {
-		$dirsize[ $directory ]['size'] = recurse_dirsize( $directory, null, $max_execution_time );
+		$size = recurse_dirsize( $directory, null, $max_execution_time );
 	}
 
-	set_transient( 'dirsize_cache', $dirsize, HOUR_IN_SECONDS );
+	return $size;
-	return $dirsize[ $directory ]['size'];
 }
 
 /**
@@ -7591,18 +7585,32 @@ function get_dirsize( $directory, $max_execution_time = null ) {
  * @since MU (3.0.0)
  * @since 4.3.0 $exclude parameter added.
  * @since 5.2.0 $max_execution_time parameter added.
+ * @since 5.6.0 $directory_cache parameter added.
  *
  * @param string       $directory          Full path of a directory.
  * @param string|array $exclude            Optional. Full path of a subdirectory to exclude from the total,
  *                                         or array of paths. Expected without trailing slash(es).
  * @param int          $max_execution_time Maximum time to run before giving up. In seconds. The timeout is global
  *                                         and is measured from the moment WordPress started to load.
+ * @param array        $directory_cache    Optional. Array of cached directory paths.
+ *
  * @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout.
  */
-function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null ) {
+function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null, &$directory_cache = null ) {
 	$size = 0;
 
 	$directory = untrailingslashit( $directory );
+	$cache_path = normalize_dirsize_cache_path( $directory );
+	$save_cache = false;
+
+	if ( ! isset( $directory_cache ) ) {
+		$directory_cache = get_transient( 'dirsize_cache' );
+		$save_cache      = true;
+	}
+
+	if ( isset( $directory_cache[ $cache_path ] ) ) {
+		return $directory_cache[ $cache_path ];
+	}
 
 	if ( ! file_exists( $directory ) || ! is_dir( $directory ) || ! is_readable( $directory ) ) {
 		return false;
@@ -7630,32 +7638,96 @@ function recurse_dirsize( $directory, $exclude = null, $max_execution_time = nul
 		}
 	}
 
-	$handle = opendir( $directory );
+	/**
-	if ( $handle ) {
+	* Filters the amount of storage space used by one directory and all it's children, in megabytes.
-		while ( ( $file = readdir( $handle ) ) !== false ) {
+	* Return the actual used space to shortcircuit the recursive PHP file size calculation and use something else
-			$path = $directory . '/' . $file;
+	* like a CDN API or native operating system tools for better performance
-			if ( '.' !== $file && '..' !== $file ) {
+	*
-				if ( is_file( $path ) ) {
+	* @since 5.6.0
-					$size += filesize( $path );
+	*
-				} elseif ( is_dir( $path ) ) {
+	* @param int|false $space_used The amount of used space, in bytes. Default 0.
-					$handlesize = recurse_dirsize( $path, $exclude, $max_execution_time );
+	*/
-					if ( $handlesize > 0 ) {
+	$size = apply_filters( 'calculate_current_dirsize', $size, $directory, $exclude, $max_execution_time, $directory_cache );
-						$size += $handlesize;
+
+	if ( 0 === $size ) {
+		$handle = opendir( $directory );
+		if ( $handle ) {
+			while ( ( $file = readdir( $handle ) ) !== false ) {
+				$path = $directory . '/' . $file;
+				if ( '.' !== $file && '..' !== $file ) {
+					if ( is_file( $path ) ) {
+						$size += filesize( $path );
+					} elseif ( is_dir( $path ) ) {
+						$handlesize = recurse_dirsize( $path, $exclude, $max_execution_time, $directory_cache );
+						if ( $handlesize > 0 ) {
+							$size += $handlesize;
+						}
+					}
+
+					if ( $max_execution_time > 0 && microtime( true ) - WP_START_TIMESTAMP > $max_execution_time ) {
+						// Time exceeded. Give up instead of risking a fatal timeout.
+						$size = null;
+						break;
 					}
 				}
-
-				if ( $max_execution_time > 0 && microtime( true ) - WP_START_TIMESTAMP > $max_execution_time ) {
-					// Time exceeded. Give up instead of risking a fatal timeout.
-					$size = null;
-					break;
-				}
 			}
+			closedir( $handle );
 		}
-		closedir( $handle );
 	}
+	$directory_cache[ $cache_path ] = $size;
+
+	// Only write the transient on the top level call and not on recursive calls
+	if ( $save_cache ) {
+		set_transient( 'dirsize_cache', $directory_cache );
+	}
+
 	return $size;
 }
 
+/**
+ * Invalidates entries within the dirsize_cache
+ *
+ * Remove the current directory and all parent directories
+ * from the dirsize_cache transient.
+ *
+ * @since 5.6.0
+ *
+ * @param string $path Full path of a directory or file.
+ */
+function invalidate_dirsize_cache( $path ) {
+	$directory_cache = get_transient( 'dirsize_cache' );
+
+	if ( empty( $directory_cache ) ) {
+		return;
+	}
+
+	$cache_path = normalize_dirsize_cache_path( $path );
+	unset( $directory_cache[ $cache_path ] );
+
+	while ( DIRECTORY_SEPARATOR !== $cache_path && '.' !== $cache_path && '..' !== $cache_path ) {
+		$cache_path = dirname( $cache_path );
+		unset( $directory_cache[ $cache_path ] );
+	}
+
+	set_transient( 'dirsize_cache', $directory_cache );
+}
+
+/**
+ * Normalize dirsize cache path.
+ *
+ * Ensures array keys within the dirsize_cache transient follow the same format.
+ *
+ * @since 5.6.0
+ *
+ * @param string $path
+ * @return string
+ */
+function normalize_dirsize_cache_path( $path ) {
+	$path = str_replace( ABSPATH, '', $path );
+
+	return untrailingslashit( $path );
+}
+
 /**
  * Checks compatibility with the current WordPress version.
  *

src/wp-includes/post.php

@@ -5914,7 +5914,7 @@ function wp_delete_attachment( $post_id, $force_delete = false ) {
 	$file         = get_attached_file( $post_id );
 
 	if ( is_multisite() ) {
-		delete_transient( 'dirsize_cache' );
+		invalidate_dirsize_cache( $file );
 	}
 
 	/**

tests/phpunit/tests/multisite/dirsizeCache.php

@@ -0,0 +1,231 @@
+<?php
+
+if ( is_multisite() ) :
+
+	/**
+	 * Tests specific to the dirsize caching in multisites
+	 *
+	 * @group multisite
+	 */
+	class Tests_Multisite_Dirsize_Cache extends WP_UnitTestCase {
+		protected $suppress = false;
+
+		function setUp() {
+			global $wpdb;
+			parent::setUp();
+			$this->suppress = $wpdb->suppress_errors();
+		}
+
+		function tearDown() {
+			global $wpdb;
+			$wpdb->suppress_errors( $this->suppress );
+			parent::tearDown();
+		}
+
+		/*
+		 * Test whether the values from the dirsize_cache will be used correctly using a more complex dirsize cache mock
+		 */
+		function test_get_dirsize_cache_in_recurse_dirsize_mock() {
+			$blog_id = self::factory()->blog->create();
+			switch_to_blog( $blog_id );
+
+			// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
+			// src directory already contains a content directory with site content, then the initial expectation
+			// will be polluted. We create sites until an empty one is available.
+			while ( 0 !== get_space_used() ) {
+				restore_current_blog();
+				$blog_id = self::factory()->blog->create();
+				switch_to_blog( $blog_id );
+			}
+
+			// Clear the dirsize_cache
+			delete_transient( 'dirsize_cache' );
+
+			// Set the dirsize cache to our mock
+			set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) );
+
+			$upload_dir = wp_upload_dir();
+
+			// Check recurse_dirsize against the mock. The cache should match
+			$this->assertSame( 21, recurse_dirsize( $upload_dir['basedir'] . '/2/1' ) );
+			$this->assertSame( 22, recurse_dirsize( $upload_dir['basedir'] . '/2/2' ) );
+			$this->assertSame( 2, recurse_dirsize( $upload_dir['basedir'] . '/2' ) );
+			$this->assertSame( 11, recurse_dirsize( $upload_dir['basedir'] . '/1/1' ) );
+			$this->assertSame( 12, recurse_dirsize( $upload_dir['basedir'] . '/1/2' ) );
+			$this->assertSame( 13, recurse_dirsize( $upload_dir['basedir'] . '/1/3' ) );
+			$this->assertSame( 1, recurse_dirsize( $upload_dir['basedir'] . '/1' ) );
+			$this->assertSame( 42, recurse_dirsize( $upload_dir['basedir'] . '/custom_directory' ) );
+
+			// No cache match, upload folder should be empty and return 0
+			$this->assertSame( 0, recurse_dirsize( $upload_dir['basedir'] ) );
+
+			// No cache match on non existing folder should return false
+			$this->assertSame( false, recurse_dirsize( $upload_dir['basedir'] . '/does_not_exist' ) );
+
+			// Cleanup
+			$this->remove_added_uploads();
+			restore_current_blog();
+		}
+
+		/*
+		 * Test whether the invalidation of the dirsize_cache works
+		 * Given a file path as input
+		 */
+		function test_invalidate_dirsize_cache_file_input_mock() {
+			$blog_id = self::factory()->blog->create();
+			switch_to_blog( $blog_id );
+
+			// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
+			// src directory already contains a content directory with site content, then the initial expectation
+			// will be polluted. We create sites until an empty one is available.
+			while ( 0 !== get_space_used() ) {
+				restore_current_blog();
+				$blog_id = self::factory()->blog->create();
+				switch_to_blog( $blog_id );
+			}
+
+			$upload_dir       = wp_upload_dir();
+			$cache_key_prefix = normalize_dirsize_cache_path( $upload_dir['basedir'] );
+
+			// Clear the dirsize_cache
+			delete_transient( 'dirsize_cache' );
+
+			// Set the dirsize cache to our mock
+			set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) );
+
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
+
+			// Invalidation should also respect the directory tree up
+			// Should work fine with path to folder OR file
+			invalidate_dirsize_cache( $upload_dir['basedir'] . '/2/1/file.dummy' );
+
+			$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
+			$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
+
+			// Other cache paths should not be invalidated
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
+
+			// Cleanup
+			$this->remove_added_uploads();
+			restore_current_blog();
+		}
+
+		/*
+		 * Test whether the invalidation of the dirsize_cache works
+		 * Given a folder path as input
+		 */
+		function test_invalidate_dirsize_cache_folder_input_mock() {
+			$blog_id = self::factory()->blog->create();
+			switch_to_blog( $blog_id );
+
+			// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
+			// src directory already contains a content directory with site content, then the initial expectation
+			// will be polluted. We create sites until an empty one is available.
+			while ( 0 !== get_space_used() ) {
+				restore_current_blog();
+				$blog_id = self::factory()->blog->create();
+				switch_to_blog( $blog_id );
+			}
+
+			$upload_dir       = wp_upload_dir();
+			$cache_key_prefix = normalize_dirsize_cache_path( $upload_dir['basedir'] );
+
+			// Clear the dirsize_cache
+			delete_transient( 'dirsize_cache' );
+
+			// Set the dirsize cache to our mock
+			set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) );
+
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
+
+			// Invalidation should also respect the directory tree up
+			// Should work fine with path to folder OR file
+			invalidate_dirsize_cache( $upload_dir['basedir'] . '/2/1' );
+
+			$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
+			$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
+
+			// Other cache paths should not be invalidated
+			$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
+
+			// Cleanup
+			$this->remove_added_uploads();
+			restore_current_blog();
+		}
+
+		/**
+		 * Test whether the values from the dirsize_cache will be used correctly using a simple real upload
+		 */
+		function test_get_dirsize_cache_in_recurse_dirsize_upload() {
+			$blog_id = self::factory()->blog->create();
+			switch_to_blog( $blog_id );
+
+			// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
+			// src directory already contains a content directory with site content, then the initial expectation
+			// will be polluted. We create sites until an empty one is available.
+			while ( 0 !== get_space_used() ) {
+				restore_current_blog();
+				$blog_id = self::factory()->blog->create();
+				switch_to_blog( $blog_id );
+			}
+
+			// Clear the dirsize_cache
+			delete_transient( 'dirsize_cache' );
+
+			$upload_dir = wp_upload_dir();
+
+			$this->assertSame( 0, recurse_dirsize( $upload_dir['path'] ) );
+
+			// Upload a file to the new site using wp_upload_bits.
+			$filename = __FUNCTION__ . '.jpg';
+			$contents = __FUNCTION__ . '_contents';
+			$file     = wp_upload_bits( $filename, null, $contents );
+
+			$calc_size = recurse_dirsize( $upload_dir['path'] );
+			$size      = filesize( $file['file'] );
+			$this->assertSame( $size, $calc_size );
+
+			// dirsize_cache should now be filled after upload and recurse_dirsize call
+			$cache_path = normalize_dirsize_cache_path( $upload_dir['path'] );
+			$this->assertSame( true, is_array( get_transient( 'dirsize_cache' ) ) );
+			$this->assertSame( $size, get_transient( 'dirsize_cache' )[ $cache_path ] );
+
+			// Cleanup
+			$this->remove_added_uploads();
+			restore_current_blog();
+		}
+
+		/*
+		 * Test whether the filter to calculate space for an existing directory works as expected
+		 */
+		function test_recurse_dirsize_calculate_current_dirsize_filter() {
+			add_filter( 'calculate_current_dirsize', array( $this, '_filter_calculate_current_dirsize' ) );
+
+			$upload_dir = wp_upload_dir();
+			$this->assertSame( 1042, recurse_dirsize( $upload_dir['path'] ) );
+
+			remove_filter( 'calculate_current_dirsize', array( $this, '_filter_calculate_current_dirsize' ) );
+		}
+
+		function _filter_calculate_current_dirsize() {
+			return 1042;
+		}
+
+		function _get_mock_dirsize_cache_for_site( $site_id ) {
+			return array(
+				"wp-content/uploads/sites/$site_id/2/2" => 22,
+				"wp-content/uploads/sites/$site_id/2/1" => 21,
+				"wp-content/uploads/sites/$site_id/2"   => 2,
+				"wp-content/uploads/sites/$site_id/1/3" => 13,
+				"wp-content/uploads/sites/$site_id/1/2" => 12,
+				"wp-content/uploads/sites/$site_id/1/1" => 11,
+				"wp-content/uploads/sites/$site_id/1"   => 1,
+				"wp-content/uploads/sites/$site_id/custom_directory" => 42,
+			);
+		}
+	}
+endif;