sergiotarxz/Wordpress
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Multisite: More specific caching for get_dirsize.

Browse Source 
Instead of one cache entry for all upload folders for a site on multisite, this now caches for each folder and invalidates that cache based on context. In multisite, this should speed up `get_dirsize` calls since older directories that are much less likely to change will no longer have the size recalculated.

Props janthiel, A5hleyRich, batmoo.
Fixes #19879.


git-svn-id: https://develop.svn.wordpress.org/trunk@49212 602fd350-edb4-49c9-b593-d223f7449a82
This commit is contained in:
Helen Hou-Sandi 2020-10-19 21:49:58 +00:00
parent 052ac08385
commit 80329c379a
4 changed files with 337 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/wp-admin/includes/file.php
View File

@ -928,7 +928,7 @@ function _wp_handle_upload( &$file, $overrides, $time, $action ) {
$url = $uploads['url'] . "/$filename";
if ( is_multisite() ) {
delete_transient( 'dirsize_cache' );
invalidate_dirsize_cache( $new_file );
}
/**

136
src/wp-includes/functions.php
View File

@ -2740,6 +2740,10 @@ function wp_upload_bits( $name, $deprecated, $bits, $time = null ) {
// Compute the URL.
$url = $upload['url'] . "/$filename";
if ( is_multisite() ) {
invalidate_dirsize_cache( $new_file );
}
/** This filter is documented in wp-admin/includes/file.php */
return apply_filters(
'wp_handle_upload',
@ -7560,26 +7564,16 @@ function wp_direct_php_update_button() {
* @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout.
*/
function get_dirsize( $directory, $max_execution_time = null ) {
$dirsize = get_transient( 'dirsize_cache' );
if ( is_array( $dirsize ) && isset( $dirsize[ $directory ]['size'] ) ) {
return $dirsize[ $directory ]['size'];
}
if ( ! is_array( $dirsize ) ) {
$dirsize = array();
}
// Exclude individual site directories from the total when checking the main site of a network,
// as they are subdirectories and should not be counted.
if ( is_multisite() && is_main_site() ) {
$dirsize[ $directory ]['size'] = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time );
$size = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time );
} else {
$dirsize[ $directory ]['size'] = recurse_dirsize( $directory, null, $max_execution_time );
$size = recurse_dirsize( $directory, null, $max_execution_time );
}
set_transient( 'dirsize_cache', $dirsize, HOUR_IN_SECONDS );
return $dirsize[ $directory ]['size'];
return $size;
}
/**
@ -7591,18 +7585,32 @@ function get_dirsize( $directory, $max_execution_time = null ) {
* @since MU (3.0.0)
* @since 4.3.0 $exclude parameter added.
* @since 5.2.0 $max_execution_time parameter added.
* @since 5.6.0 $directory_cache parameter added.
*
* @param string $directory Full path of a directory.
* @param string|array $exclude Optional. Full path of a subdirectory to exclude from the total,
* or array of paths. Expected without trailing slash(es).
* @param int $max_execution_time Maximum time to run before giving up. In seconds. The timeout is global
* and is measured from the moment WordPress started to load.
* @param array $directory_cache Optional. Array of cached directory paths.
*
* @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout.
*/
function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null ) {
function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null, &$directory_cache = null ) {
$size = 0;
$directory = untrailingslashit( $directory );
$cache_path = normalize_dirsize_cache_path( $directory );
$save_cache = false;
if ( ! isset( $directory_cache ) ) {
$directory_cache = get_transient( 'dirsize_cache' );
$save_cache = true;
}
if ( isset( $directory_cache[ $cache_path ] ) ) {
return $directory_cache[ $cache_path ];
}
if ( ! file_exists( $directory ) || ! is_dir( $directory ) || ! is_readable( $directory ) ) {
return false;
@ -7630,32 +7638,96 @@ function recurse_dirsize( $directory, $exclude = null, $max_execution_time = nul
}
}
$handle = opendir( $directory );
if ( $handle ) {
while ( ( $file = readdir( $handle ) ) !== false ) {
$path = $directory . '/' . $file;
if ( '.' !== $file && '..' !== $file ) {
if ( is_file( $path ) ) {
$size += filesize( $path );
} elseif ( is_dir( $path ) ) {
$handlesize = recurse_dirsize( $path, $exclude, $max_execution_time );
if ( $handlesize > 0 ) {
$size += $handlesize;
/**
* Filters the amount of storage space used by one directory and all it's children, in megabytes.
* Return the actual used space to shortcircuit the recursive PHP file size calculation and use something else
* like a CDN API or native operating system tools for better performance
*
* @since 5.6.0
*
* @param int|false $space_used The amount of used space, in bytes. Default 0.
*/
$size = apply_filters( 'calculate_current_dirsize', $size, $directory, $exclude, $max_execution_time, $directory_cache );
if ( 0 === $size ) {
$handle = opendir( $directory );
if ( $handle ) {
while ( ( $file = readdir( $handle ) ) !== false ) {
$path = $directory . '/' . $file;
if ( '.' !== $file && '..' !== $file ) {
if ( is_file( $path ) ) {
$size += filesize( $path );
} elseif ( is_dir( $path ) ) {
$handlesize = recurse_dirsize( $path, $exclude, $max_execution_time, $directory_cache );
if ( $handlesize > 0 ) {
$size += $handlesize;
}
}
if ( $max_execution_time > 0 && microtime( true ) - WP_START_TIMESTAMP > $max_execution_time ) {
// Time exceeded. Give up instead of risking a fatal timeout.
$size = null;
break;
}
}
if ( $max_execution_time > 0 && microtime( true ) - WP_START_TIMESTAMP > $max_execution_time ) {
// Time exceeded. Give up instead of risking a fatal timeout.
$size = null;
break;
}
}
closedir( $handle );
}
closedir( $handle );
}
$directory_cache[ $cache_path ] = $size;
// Only write the transient on the top level call and not on recursive calls
if ( $save_cache ) {
set_transient( 'dirsize_cache', $directory_cache );
}
return $size;
}
/**
* Invalidates entries within the dirsize_cache
*
* Remove the current directory and all parent directories
* from the dirsize_cache transient.
*
* @since 5.6.0
*
* @param string $path Full path of a directory or file.
*/
function invalidate_dirsize_cache( $path ) {
$directory_cache = get_transient( 'dirsize_cache' );
if ( empty( $directory_cache ) ) {
return;
}
$cache_path = normalize_dirsize_cache_path( $path );
unset( $directory_cache[ $cache_path ] );
while ( DIRECTORY_SEPARATOR !== $cache_path && '.' !== $cache_path && '..' !== $cache_path ) {
$cache_path = dirname( $cache_path );
unset( $directory_cache[ $cache_path ] );
}
set_transient( 'dirsize_cache', $directory_cache );
}
/**
* Normalize dirsize cache path.
*
* Ensures array keys within the dirsize_cache transient follow the same format.
*
* @since 5.6.0
*
* @param string $path
* @return string
*/
function normalize_dirsize_cache_path( $path ) {
$path = str_replace( ABSPATH, '', $path );
return untrailingslashit( $path );
}
/**
* Checks compatibility with the current WordPress version.
*

2
src/wp-includes/post.php
View File

@ -5914,7 +5914,7 @@ function wp_delete_attachment( $post_id, $force_delete = false ) {
$file = get_attached_file( $post_id );
if ( is_multisite() ) {
delete_transient( 'dirsize_cache' );
invalidate_dirsize_cache( $file );
}
/**

231
tests/phpunit/tests/multisite/dirsizeCache.php Normal file
View File

@ -0,0 +1,231 @@
<?php
if ( is_multisite() ) :
/**
* Tests specific to the dirsize caching in multisites
*
* @group multisite
*/
class Tests_Multisite_Dirsize_Cache extends WP_UnitTestCase {
protected $suppress = false;
function setUp() {
global $wpdb;
parent::setUp();
$this->suppress = $wpdb->suppress_errors();
}
function tearDown() {
global $wpdb;
$wpdb->suppress_errors( $this->suppress );
parent::tearDown();
}
/*
* Test whether the values from the dirsize_cache will be used correctly using a more complex dirsize cache mock
*/
function test_get_dirsize_cache_in_recurse_dirsize_mock() {
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
// src directory already contains a content directory with site content, then the initial expectation
// will be polluted. We create sites until an empty one is available.
while ( 0 !== get_space_used() ) {
restore_current_blog();
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
}
// Clear the dirsize_cache
delete_transient( 'dirsize_cache' );
// Set the dirsize cache to our mock
set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) );
$upload_dir = wp_upload_dir();
// Check recurse_dirsize against the mock. The cache should match
$this->assertSame( 21, recurse_dirsize( $upload_dir['basedir'] . '/2/1' ) );
$this->assertSame( 22, recurse_dirsize( $upload_dir['basedir'] . '/2/2' ) );
$this->assertSame( 2, recurse_dirsize( $upload_dir['basedir'] . '/2' ) );
$this->assertSame( 11, recurse_dirsize( $upload_dir['basedir'] . '/1/1' ) );
$this->assertSame( 12, recurse_dirsize( $upload_dir['basedir'] . '/1/2' ) );
$this->assertSame( 13, recurse_dirsize( $upload_dir['basedir'] . '/1/3' ) );
$this->assertSame( 1, recurse_dirsize( $upload_dir['basedir'] . '/1' ) );
$this->assertSame( 42, recurse_dirsize( $upload_dir['basedir'] . '/custom_directory' ) );
// No cache match, upload folder should be empty and return 0
$this->assertSame( 0, recurse_dirsize( $upload_dir['basedir'] ) );
// No cache match on non existing folder should return false
$this->assertSame( false, recurse_dirsize( $upload_dir['basedir'] . '/does_not_exist' ) );
// Cleanup
$this->remove_added_uploads();
restore_current_blog();
}
/*
* Test whether the invalidation of the dirsize_cache works
* Given a file path as input
*/
function test_invalidate_dirsize_cache_file_input_mock() {
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
// src directory already contains a content directory with site content, then the initial expectation
// will be polluted. We create sites until an empty one is available.
while ( 0 !== get_space_used() ) {
restore_current_blog();
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
}
$upload_dir = wp_upload_dir();
$cache_key_prefix = normalize_dirsize_cache_path( $upload_dir['basedir'] );
// Clear the dirsize_cache
delete_transient( 'dirsize_cache' );
// Set the dirsize cache to our mock
set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) );
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
// Invalidation should also respect the directory tree up
// Should work fine with path to folder OR file
invalidate_dirsize_cache( $upload_dir['basedir'] . '/2/1/file.dummy' );
$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
// Other cache paths should not be invalidated
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
// Cleanup
$this->remove_added_uploads();
restore_current_blog();
}
/*
* Test whether the invalidation of the dirsize_cache works
* Given a folder path as input
*/
function test_invalidate_dirsize_cache_folder_input_mock() {
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
// src directory already contains a content directory with site content, then the initial expectation
// will be polluted. We create sites until an empty one is available.
while ( 0 !== get_space_used() ) {
restore_current_blog();
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
}
$upload_dir = wp_upload_dir();
$cache_key_prefix = normalize_dirsize_cache_path( $upload_dir['basedir'] );
// Clear the dirsize_cache
delete_transient( 'dirsize_cache' );
// Set the dirsize cache to our mock
set_transient( 'dirsize_cache', $this->_get_mock_dirsize_cache_for_site( $blog_id ) );
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
// Invalidation should also respect the directory tree up
// Should work fine with path to folder OR file
invalidate_dirsize_cache( $upload_dir['basedir'] . '/2/1' );
$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2/1', get_transient( 'dirsize_cache' ) ) );
$this->assertSame( false, array_key_exists( $cache_key_prefix . '/2', get_transient( 'dirsize_cache' ) ) );
// Other cache paths should not be invalidated
$this->assertSame( true, array_key_exists( $cache_key_prefix . '/1/1', get_transient( 'dirsize_cache' ) ) );
// Cleanup
$this->remove_added_uploads();
restore_current_blog();
}
/**
* Test whether the values from the dirsize_cache will be used correctly using a simple real upload
*/
function test_get_dirsize_cache_in_recurse_dirsize_upload() {
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
// Our comparison of space relies on an initial value of 0. If a previous test has failed or if the
// src directory already contains a content directory with site content, then the initial expectation
// will be polluted. We create sites until an empty one is available.
while ( 0 !== get_space_used() ) {
restore_current_blog();
$blog_id = self::factory()->blog->create();
switch_to_blog( $blog_id );
}
// Clear the dirsize_cache
delete_transient( 'dirsize_cache' );
$upload_dir = wp_upload_dir();
$this->assertSame( 0, recurse_dirsize( $upload_dir['path'] ) );
// Upload a file to the new site using wp_upload_bits.
$filename = __FUNCTION__ . '.jpg';
$contents = __FUNCTION__ . '_contents';
$file = wp_upload_bits( $filename, null, $contents );
$calc_size = recurse_dirsize( $upload_dir['path'] );
$size = filesize( $file['file'] );
$this->assertSame( $size, $calc_size );
// dirsize_cache should now be filled after upload and recurse_dirsize call
$cache_path = normalize_dirsize_cache_path( $upload_dir['path'] );
$this->assertSame( true, is_array( get_transient( 'dirsize_cache' ) ) );
$this->assertSame( $size, get_transient( 'dirsize_cache' )[ $cache_path ] );
// Cleanup
$this->remove_added_uploads();
restore_current_blog();
}
/*
* Test whether the filter to calculate space for an existing directory works as expected
*/
function test_recurse_dirsize_calculate_current_dirsize_filter() {
add_filter( 'calculate_current_dirsize', array( $this, '_filter_calculate_current_dirsize' ) );
$upload_dir = wp_upload_dir();
$this->assertSame( 1042, recurse_dirsize( $upload_dir['path'] ) );
remove_filter( 'calculate_current_dirsize', array( $this, '_filter_calculate_current_dirsize' ) );
}
function _filter_calculate_current_dirsize() {
return 1042;
}
function _get_mock_dirsize_cache_for_site( $site_id ) {
return array(
"wp-content/uploads/sites/$site_id/2/2" => 22,
"wp-content/uploads/sites/$site_id/2/1" => 21,
"wp-content/uploads/sites/$site_id/2" => 2,
"wp-content/uploads/sites/$site_id/1/3" => 13,
"wp-content/uploads/sites/$site_id/1/2" => 12,
"wp-content/uploads/sites/$site_id/1/1" => 11,
"wp-content/uploads/sites/$site_id/1" => 1,
"wp-content/uploads/sites/$site_id/custom_directory" => 42,
);
}
}
endif;