From 80329c379a89a53a1814258ee8b1b3c8aa6bf46b Mon Sep 17 00:00:00 2001 From: Helen Hou-Sandi Date: Mon, 19 Oct 2020 21:49:58 +0000 Subject: [PATCH] Multisite: More specific caching for `get_dirsize`. Instead of one cache entry for all upload folders for a site on multisite, this now caches for each folder and invalidates that cache based on context. In multisite, this should speed up `get_dirsize` calls since older directories that are much less likely to change will no longer have the size recalculated. Props janthiel, A5hleyRich, batmoo. Fixes #19879. git-svn-id: https://develop.svn.wordpress.org/trunk@49212 602fd350-edb4-49c9-b593-d223f7449a82 --- src/wp-admin/includes/file.php | 2 +- src/wp-includes/functions.php | 136 ++++++++--- src/wp-includes/post.php | 2 +- .../phpunit/tests/multisite/dirsizeCache.php | 231 ++++++++++++++++++ 4 files changed, 337 insertions(+), 34 deletions(-) create mode 100644 tests/phpunit/tests/multisite/dirsizeCache.php diff --git a/src/wp-admin/includes/file.php b/src/wp-admin/includes/file.php index cfb40529a6..04dea32a20 100644 --- a/src/wp-admin/includes/file.php +++ b/src/wp-admin/includes/file.php @@ -928,7 +928,7 @@ function _wp_handle_upload( &$file, $overrides, $time, $action ) { $url = $uploads['url'] . "/$filename"; if ( is_multisite() ) { - delete_transient( 'dirsize_cache' ); + invalidate_dirsize_cache( $new_file ); } /** diff --git a/src/wp-includes/functions.php b/src/wp-includes/functions.php index 0d034a5b08..8b36b9b09b 100644 --- a/src/wp-includes/functions.php +++ b/src/wp-includes/functions.php @@ -2740,6 +2740,10 @@ function wp_upload_bits( $name, $deprecated, $bits, $time = null ) { // Compute the URL. $url = $upload['url'] . "/$filename"; + if ( is_multisite() ) { + invalidate_dirsize_cache( $new_file ); + } + /** This filter is documented in wp-admin/includes/file.php */ return apply_filters( 'wp_handle_upload', @@ -7560,26 +7564,16 @@ function wp_direct_php_update_button() { * @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout. */ function get_dirsize( $directory, $max_execution_time = null ) { - $dirsize = get_transient( 'dirsize_cache' ); - - if ( is_array( $dirsize ) && isset( $dirsize[ $directory ]['size'] ) ) { - return $dirsize[ $directory ]['size']; - } - - if ( ! is_array( $dirsize ) ) { - $dirsize = array(); - } // Exclude individual site directories from the total when checking the main site of a network, // as they are subdirectories and should not be counted. if ( is_multisite() && is_main_site() ) { - $dirsize[ $directory ]['size'] = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time ); + $size = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time ); } else { - $dirsize[ $directory ]['size'] = recurse_dirsize( $directory, null, $max_execution_time ); + $size = recurse_dirsize( $directory, null, $max_execution_time ); } - set_transient( 'dirsize_cache', $dirsize, HOUR_IN_SECONDS ); - return $dirsize[ $directory ]['size']; + return $size; } /** @@ -7591,18 +7585,32 @@ function get_dirsize( $directory, $max_execution_time = null ) { * @since MU (3.0.0) * @since 4.3.0 $exclude parameter added. * @since 5.2.0 $max_execution_time parameter added. + * @since 5.6.0 $directory_cache parameter added. * * @param string $directory Full path of a directory. * @param string|array $exclude Optional. Full path of a subdirectory to exclude from the total, * or array of paths. Expected without trailing slash(es). * @param int $max_execution_time Maximum time to run before giving up. In seconds. The timeout is global * and is measured from the moment WordPress started to load. + * @param array $directory_cache Optional. Array of cached directory paths. + * * @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout. */ -function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null ) { +function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null, &$directory_cache = null ) { $size = 0; $directory = untrailingslashit( $directory ); + $cache_path = normalize_dirsize_cache_path( $directory ); + $save_cache = false; + + if ( ! isset( $directory_cache ) ) { + $directory_cache = get_transient( 'dirsize_cache' ); + $save_cache = true; + } + + if ( isset( $directory_cache[ $cache_path ] ) ) { + return $directory_cache[ $cache_path ]; + } if ( ! file_exists( $directory ) || ! is_dir( $directory ) || ! is_readable( $directory ) ) { return false; @@ -7630,32 +7638,96 @@ function recurse_dirsize( $directory, $exclude = null, $max_execution_time = nul } } - $handle = opendir( $directory ); - if ( $handle ) { - while ( ( $file = readdir( $handle ) ) !== false ) { - $path = $directory . '/' . $file; - if ( '.' !== $file && '..' !== $file ) { - if ( is_file( $path ) ) { - $size += filesize( $path ); - } elseif ( is_dir( $path ) ) { - $handlesize = recurse_dirsize( $path, $exclude, $max_execution_time ); - if ( $handlesize > 0 ) { - $size += $handlesize; + /** + * Filters the amount of storage space used by one directory and all it's children, in megabytes. + * Return the actual used space to shortcircuit the recursive PHP file size calculation and use something else + * like a CDN API or native operating system tools for better performance + * + * @since 5.6.0 + * + * @param int|false $space_used The amount of used space, in bytes. Default 0. + */ + $size = apply_filters( 'calculate_current_dirsize', $size, $directory, $exclude, $max_execution_time, $directory_cache ); + + if ( 0 === $size ) { + $handle = opendir( $directory ); + if ( $handle ) { + while ( ( $file = readdir( $handle ) ) !== false ) { + $path = $directory . '/' . $file; + if ( '.' !== $file && '..' !== $file ) { + if ( is_file( $path ) ) { + $size += filesize( $path ); + } elseif ( is_dir( $path ) ) { + $handlesize = recurse_dirsize( $path, $exclude, $max_execution_time, $directory_cache ); + if ( $handlesize > 0 ) { + $size += $handlesize; + } + } + + if ( $max_execution_time > 0 && microtime( true ) - WP_START_TIMESTAMP > $max_execution_time ) { + // Time exceeded. Give up instead of risking a fatal timeout. + $size = null; + break; } } - - if ( $max_execution_time > 0 && microtime( true ) - WP_START_TIMESTAMP > $max_execution_time ) { - // Time exceeded. Give up instead of risking a fatal timeout. - $size = null; - break; - } } + closedir( $handle ); } - closedir( $handle ); } + $directory_cache[ $cache_path ] = $size; + + // Only write the transient on the top level call and not on recursive calls + if ( $save_cache ) { + set_transient( 'dirsize_cache', $directory_cache ); + } + return $size; } +/** + * Invalidates entries within the dirsize_cache + * + * Remove the current directory and all parent directories + * from the dirsize_cache transient. + * + * @since 5.6.0 + * + * @param string $path Full path of a directory or file. + */ +function invalidate_dirsize_cache( $path ) { + $directory_cache = get_transient( 'dirsize_cache' ); + + if ( empty( $directory_cache ) ) { + return; + } + + $cache_path = normalize_dirsize_cache_path( $path ); + unset( $directory_cache[ $cache_path ] ); + + while ( DIRECTORY_SEPARATOR !== $cache_path && '.' !== $cache_path && '..' !== $cache_path ) { + $cache_path = dirname( $cache_path ); + unset( $directory_cache[ $cache_path ] ); + } + + set_transient( 'dirsize_cache', $directory_cache ); +} + +/** + * Normalize dirsize cache path. + * + * Ensures array keys within the dirsize_cache transient follow the same format. + * + * @since 5.6.0 + * + * @param string $path + * @return string + */ +function normalize_dirsize_cache_path( $path ) { + $path = str_replace( ABSPATH, '', $path ); + + return untrailingslashit( $path ); +} + /** * Checks compatibility with the current WordPress version. * diff --git a/src/wp-includes/post.php b/src/wp-includes/post.php index 6e968f3f9f..b44bf573cc 100644 --- a/src/wp-includes/post.php +++ b/src/wp-includes/post.php @@ -5914,7 +5914,7 @@ function wp_delete_attachment( $post_id, $force_delete = false ) { $file = get_attached_file( $post_id ); if ( is_multisite() ) { - delete_transient( 'dirsize_cache' ); + invalidate_dirsize_cache( $file ); } /** diff --git a/tests/phpunit/tests/multisite/dirsizeCache.php b/tests/phpunit/tests/multisite/dirsizeCache.php new file mode 100644 index 0000000000..c9bdb83c00 --- /dev/null +++ b/tests/phpunit/tests/multisite/dirsizeCache.php @@ -0,0 +1,231 @@ +suppress = $wpdb->suppress_errors(); + } + + function tearDown() { + global $wpdb; + $wpdb->suppress_errors( $this->suppress ); + parent::tearDown(); + } + + /* + * Test whether the values from the dirsize_cache will be used correctly using a more complex dirsize cache mock + */ + function test_get_dirsize_cache_in_recurse_dirsize_mock() { + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + + // Our comparison of space relies on an initial value of 0. If a previous test has failed or if the + // src directory already contains a content directory with site content, then the initial expectation + // will be polluted. We create sites until an empty one is available. + while ( 0 !== get_space_used() ) { + restore_current_blog(); + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + } + + // Clear the dirsize_cache + delete_transient( 'dirsize_cache' ); + + // Set the dirsize cache to our mock + set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) ); + + $upload_dir = wp_upload_dir(); + + // Check recurse_dirsize against the mock. The cache should match + $this->assertSame( 21, recurse_dirsize( $upload_dir['basedir'] . '/2/1' ) ); + $this->assertSame( 22, recurse_dirsize( $upload_dir['basedir'] . '/2/2' ) ); + $this->assertSame( 2, recurse_dirsize( $upload_dir['basedir'] . '/2' ) ); + $this->assertSame( 11, recurse_dirsize( $upload_dir['basedir'] . '/1/1' ) ); + $this->assertSame( 12, recurse_dirsize( $upload_dir['basedir'] . '/1/2' ) ); + $this->assertSame( 13, recurse_dirsize( $upload_dir['basedir'] . '/1/3' ) ); + $this->assertSame( 1, recurse_dirsize( $upload_dir['basedir'] . '/1' ) ); + $this->assertSame( 42, recurse_dirsize( $upload_dir['basedir'] . '/custom_directory' ) ); + + // No cache match, upload folder should be empty and return 0 + $this->assertSame( 0, recurse_dirsize( $upload_dir['basedir'] ) ); + + // No cache match on non existing folder should return false + $this->assertSame( false, recurse_dirsize( $upload_dir['basedir'] . '/does_not_exist' ) ); + + // Cleanup + $this->remove_added_uploads(); + restore_current_blog(); + } + + /* + * Test whether the invalidation of the dirsize_cache works + * Given a file path as input + */ + function test_invalidate_dirsize_cache_file_input_mock() { + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + + // Our comparison of space relies on an initial value of 0. If a previous test has failed or if the + // src directory already contains a content directory with site content, then the initial expectation + // will be polluted. We create sites until an empty one is available. + while ( 0 !== get_space_used() ) { + restore_current_blog(); + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + } + + $upload_dir = wp_upload_dir(); + $cache_key_prefix = normalize_dirsize_cache_path( $upload_dir['basedir'] ); + + // Clear the dirsize_cache + delete_transient( 'dirsize_cache' ); + + // Set the dirsize cache to our mock + set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) ); + + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) ); + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) ); + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) ); + + // Invalidation should also respect the directory tree up + // Should work fine with path to folder OR file + invalidate_dirsize_cache( $upload_dir['basedir'] . '/2/1/file.dummy' ); + + $this->assertSame( false, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) ); + $this->assertSame( false, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) ); + + // Other cache paths should not be invalidated + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) ); + + // Cleanup + $this->remove_added_uploads(); + restore_current_blog(); + } + + /* + * Test whether the invalidation of the dirsize_cache works + * Given a folder path as input + */ + function test_invalidate_dirsize_cache_folder_input_mock() { + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + + // Our comparison of space relies on an initial value of 0. If a previous test has failed or if the + // src directory already contains a content directory with site content, then the initial expectation + // will be polluted. We create sites until an empty one is available. + while ( 0 !== get_space_used() ) { + restore_current_blog(); + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + } + + $upload_dir = wp_upload_dir(); + $cache_key_prefix = normalize_dirsize_cache_path( $upload_dir['basedir'] ); + + // Clear the dirsize_cache + delete_transient( 'dirsize_cache' ); + + // Set the dirsize cache to our mock + set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) ); + + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) ); + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) ); + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) ); + + // Invalidation should also respect the directory tree up + // Should work fine with path to folder OR file + invalidate_dirsize_cache( $upload_dir['basedir'] . '/2/1' ); + + $this->assertSame( false, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) ); + $this->assertSame( false, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) ); + + // Other cache paths should not be invalidated + $this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) ); + + // Cleanup + $this->remove_added_uploads(); + restore_current_blog(); + } + + /** + * Test whether the values from the dirsize_cache will be used correctly using a simple real upload + */ + function test_get_dirsize_cache_in_recurse_dirsize_upload() { + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + + // Our comparison of space relies on an initial value of 0. If a previous test has failed or if the + // src directory already contains a content directory with site content, then the initial expectation + // will be polluted. We create sites until an empty one is available. + while ( 0 !== get_space_used() ) { + restore_current_blog(); + $blog_id = self::factory()->blog->create(); + switch_to_blog( $blog_id ); + } + + // Clear the dirsize_cache + delete_transient( 'dirsize_cache' ); + + $upload_dir = wp_upload_dir(); + + $this->assertSame( 0, recurse_dirsize( $upload_dir['path'] ) ); + + // Upload a file to the new site using wp_upload_bits. + $filename = __FUNCTION__ . '.jpg'; + $contents = __FUNCTION__ . '_contents'; + $file = wp_upload_bits( $filename, null, $contents ); + + $calc_size = recurse_dirsize( $upload_dir['path'] ); + $size = filesize( $file['file'] ); + $this->assertSame( $size, $calc_size ); + + // dirsize_cache should now be filled after upload and recurse_dirsize call + $cache_path = normalize_dirsize_cache_path( $upload_dir['path'] ); + $this->assertSame( true, is_array( get_transient( 'dirsize_cache' ) ) ); + $this->assertSame( $size, get_transient( 'dirsize_cache' )[ $cache_path ] ); + + // Cleanup + $this->remove_added_uploads(); + restore_current_blog(); + } + + /* + * Test whether the filter to calculate space for an existing directory works as expected + */ + function test_recurse_dirsize_calculate_current_dirsize_filter() { + add_filter( 'calculate_current_dirsize', array( $this, '_filter_calculate_current_dirsize' ) ); + + $upload_dir = wp_upload_dir(); + $this->assertSame( 1042, recurse_dirsize( $upload_dir['path'] ) ); + + remove_filter( 'calculate_current_dirsize', array( $this, '_filter_calculate_current_dirsize' ) ); + } + + function _filter_calculate_current_dirsize() { + return 1042; + } + + function _get_mock_dirsize_cache_for_site( $site_id ) { + return array( + "wp-content/uploads/sites/$site_id/2/2" => 22, + "wp-content/uploads/sites/$site_id/2/1" => 21, + "wp-content/uploads/sites/$site_id/2" => 2, + "wp-content/uploads/sites/$site_id/1/3" => 13, + "wp-content/uploads/sites/$site_id/1/2" => 12, + "wp-content/uploads/sites/$site_id/1/1" => 11, + "wp-content/uploads/sites/$site_id/1" => 1, + "wp-content/uploads/sites/$site_id/custom_directory" => 42, + ); + } + } +endif;