|is', '', $comment );
-
- // Parse this comment into an array
- $comment = $this->parse_comment( $comment );
- if ( empty( $comment['comment_post_ID'] ) )
- continue;
-
- // Add this comment to the appropriate cache file
- $filename = $this->full_path( 'ljimport-comments-' . $comment['comment_post_ID'] . '.php' );
- if ( $this->write_file( '' . "\n",
- $filename,
- $comment['comment_post_ID'],
- 'a' ) )
- {
- // Keep track of files used
- $cache_files[] = $filename;
- }
- }
-
- // Update list of files in the DB
- sort( $cache_files );
- $cache_files = array_unique( $cache_files );
- update_option( 'ljapi_comment_cache_files', $cache_files );
- update_option( 'ljapi_comment_cache_files_count', count( $cache_files ) );
- $this->close_file_pointers();
-
- // Don't need this XML file anymore
- unlink( $xml_file );
+ // Counter just used to show progress to user
+ update_option( 'ljapi_comment_batch', ( (int) get_option( 'ljapi_comment_batch' ) + 1 ) );
return true;
}
+ // Takes a block of XML and parses out all the elements of the comment
function parse_comment( $comment ) {
global $wpdb;
@@ -562,17 +534,19 @@ class LJ_API_Import {
preg_match( '| jitemid=\'(\d+)\'|i', $attribs[1], $matches );
$lj_comment_post_ID = $matches[1];
preg_match( '| posterid=\'(\d+)\'|i', $attribs[1], $matches );
- $comment_author_ID = $matches[1];
- preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches );
- $lj_comment_parent = $matches[1];
- preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches );
- $lj_comment_state = !empty( $matches[1] ) ? $matches[1] : 'A';
+ $comment_author_ID = isset( $matches[1] ) ? $matches[1] : 0;
+ preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches ); // optional
+ $lj_comment_parent = isset( $matches[1] ) ? $matches[1] : 0;
+ preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches ); // optional
+ $lj_comment_state = isset( $matches[1] ) ? $matches[1] : 'A';
// Clean up "subject" - this will become the first line of the comment in WP
preg_match( '|(.*)|is', $comment, $matches );
- $comment_subject = $wpdb->escape( trim( $matches[1] ) );
- if ( 'Re:' == $comment_subject )
- $comment_subject = '';
+ if ( isset( $matches[1] ) ) {
+ $comment_subject = $wpdb->escape( trim( $matches[1] ) );
+ if ( 'Re:' == $comment_subject )
+ $comment_subject = '';
+ }
// Get the body and HTMLize it
preg_match( '|(.*)|is', $comment, $matches );
@@ -590,11 +564,11 @@ class LJ_API_Import {
$comment_date = trim( str_replace( array( 'T', 'Z' ), ' ', $matches[1] ) );
// Grab IP if available
- preg_match( '|(.*)|i', $comment, $matches );
- $comment_author_IP = $matches[1];
+ preg_match( '|(.*)|i', $comment, $matches ); // optional
+ $comment_author_IP = isset( $matches[1] ) ? $matches[1] : '';
// Try to get something useful for the comment author, especially if it was "my" comment
- $author = ( substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' || empty( $comment_author_ID ) ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID];
+ $author = ( empty( $comment_author_ID ) || empty( $this->usermap[$comment_author_ID] ) || substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID];
if ( get_option( 'ljapi_username' ) == $author ) {
$user = wp_get_current_user();
$user_id = $user->ID;
@@ -613,13 +587,16 @@ class LJ_API_Import {
'comment_post_ID' => $this->get_wp_post_ID( $lj_comment_post_ID ),
'comment_author' => $author,
'comment_author_url' => $url,
+ 'comment_author_email' => '',
'comment_content' => $comment_content,
'comment_date' => $comment_date,
'comment_author_IP' => ( !empty( $comment_author_IP ) ? $comment_author_IP : '' ),
'comment_approved' => ( in_array( $lj_comment_state, array( 'A', 'F' ) ) ? 1 : 0 ),
- 'comment_agent' => 'WP LJ Importer',
- 'user_id' => $user_id
- );
+ 'comment_karma' => $lj_comment_ID, // Need this and next value until rethreading is done
+ 'comment_agent' => $lj_comment_parent,
+ 'comment_type' => 'livejournal', // Custom type, so we can find it later for processing
+ 'user_ID' => $user_id
+ );
}
@@ -627,124 +604,16 @@ class LJ_API_Import {
function get_wp_post_ID( $post ) {
global $wpdb;
if ( empty( $this->postmap[$post] ) )
- $this->postmap[$post] = $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) );
+ $this->postmap[$post] = (int) $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) );
return $this->postmap[$post];
}
- // Re-build the threading within a single cache file
- function thread_comments( $filename ) {
- if ( !is_file( $filename ) || !is_readable( $filename ) )
- return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) );
-
- $comments = array();
- @include( $filename );
- $this->comments = $comments;
- unset( $comments );
- if ( !is_array( $this->comments ) )
- $this->comments = array();
-
- $count = count( $this->comments );
- for ( $c = 0; $c < $count; $c++ ) {
- // Skip anything that's not "top-level" for now
- if ( 0 != $this->comments[$c]['lj_comment_parent'] )
- continue;
- $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] );
- }
-
- // Remove anything that's not supposed to be at top level
- $top_comments = array();
- for ( $c = 0; $c < $count; $c++ ) {
- if ( 0 == $this->comments[$c]['lj_comment_parent'] ) {
- $top_comments[] = $this->comments[$c];
- }
- }
-
- // Write back to file
- @unlink( $filename );
- $this->write_file( '', $filename, $count, 'w' );
- unset( $top_comments );
- $this->close_file_pointers();
-
- // Reference this file as being threaded
- $files = get_option( 'ljapi_comment_threaded_files' );
- $files[] = $filename;
- array_unique( $files );
- update_option( 'ljapi_comment_threaded_files', $files );
- update_option( 'ljapi_comment_threaded_files_count', count( $files ) );
-
- return true;
- }
-
- function get_child_comments( $id ) {
- $children = array();
- $count = count( $this->comments );
- for ( $c = 0; $c < $count; $c++ ) {
- // This comment is a child of the $id
- if ( $id == $this->comments[$c]['lj_comment_parent'] ) {
- $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] );
- $children[] = $this->comments[$c];
- }
- }
- return $children;
- }
-
- // Inserts the contents of each cache file (should be threaded already)
- function insert_comments( $filename ) {
- echo '';
-
- if ( !is_file( $filename ) || !is_readable( $filename ) )
- return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) );
-
- $comments = array();
- @include( $filename );
- $this->comments = $comments;
- unset( $comments );
- if ( !is_array( $this->comments ) )
- $this->comments = array();
-
- $count = count( $this->comments );
- for ( $c = 0; $c < $count; $c++ ) {
- $comment =& $this->comments[$c];
- echo '- ';
- printf( __( 'Imported comment from %s on %s' ), $comment['comment_author'], $comment['comment_date'] );
-
- $id = wp_insert_comment( $comment );
- $comment['comment_ID'] = $id;
- if ( count( $comment['children'] ) ) {
- _e( ' and replies:' );
- $this->insert_child_comments( $comment['children'], $id );
- }
-
- echo '
';
- }
-
- // Remove the file now that we're done with it
- @unlink( $filename );
-
- echo '
';
-
- return true;
- }
-
- function insert_child_comments( &$comments, $parent ) {
- echo '';
- $count = count( $comments );
- for ( $c = 0; $c < $count; $c++ ) {
- $comment =& $comments[$c];
- $comment['comment_parent'] = $parent;
- echo '- ';
- printf( __( 'Imported reply from %s on %s' ), $comment['comment_author'], $comment['comment_date'] );
-
- $id = wp_insert_comment( $comment );
- $comment['comment_ID'] = $id;
- if ( count( $comment['children'] ) ) {
- _e( ' and replies:' );
- $this->insert_child_comments( $comment['children'], $id );
- }
-
- echo '
';
- }
- echo '
';
+ // Gets the comment_ID that a LJ comment has been saved as within WP
+ function get_wp_comment_ID( $comment ) {
+ global $wpdb;
+ if ( empty( $this->commentmap[$comment] ) )
+ $this->commentmap[$comment] = $wpdb->get_var( $wpdb->prepare( "SELECT comment_ID FROM $wpdb->comments WHERE comment_karma = %d", $comment ) );
+ return $this->commentmap[$comment];
}
function lj_ixr() {
@@ -767,7 +636,6 @@ class LJ_API_Import {
if ( $this->ixr->query( 'LJ.XMLRPC.' . $method, $params ) ) {
return $this->ixr->getResponse();
} else {
- $this->log( $this->ixr->message, 'ljimport-error-' . $method . '.txt' );
return new WP_Error( 'IXR', __( 'XML-RPC Request Failed - ' ) . $this->ixr->getErrorCode() . ': ' . $this->ixr->getErrorMessage() );
}
}
@@ -788,12 +656,10 @@ class LJ_API_Import {
$this->greet();
break;
case 1 :
- case 2 :
$this->ixr = new IXR_Client( $this->ixr_url );
// Intentional no break
+ case 2 :
case 3 :
- case 4 :
- case 5 :
check_admin_referer( 'lj-api-import' );
$result = $this->{ 'step' . $step }();
if ( is_wp_error( $result ) )
@@ -837,6 +703,8 @@ class LJ_API_Import {
$login = $this->lj_ixr( 'login' );
if ( is_wp_error( $login ) ) {
if ( 100 == $this->ixr->getErrorCode() || 101 == $this->ixr->getErrorCode() ) {
+ delete_option( 'ljapi_username' );
+ delete_option( 'ljapi_password' );
?>
@@ -848,11 +716,9 @@ class LJ_API_Import {
}
// Set up some options to avoid them autoloading (these ones get big)
- add_option( 'ljapi_sync_item_times', '', '', 'no' );
- add_option( 'ljapi_usermap', '', '', 'no' );
- add_option( 'ljapi_comment_xml_files', '', '', 'no' );
- add_option( 'ljapi_comment_cache_files', '', '', 'no' );
- add_option( 'ljapi_comment_threaded_files', '', '', 'no' );
+ add_option( 'ljapi_sync_item_times', '', '', 'no' );
+ add_option( 'ljapi_usermap', '', '', 'no' );
+ update_option( 'ljapi_comment_batch', 0 );
echo '' . __( 'Importing Posts' ) . '
';
echo '' . __( "We're downloading and importing all your LiveJournal posts..." ) . '
';
@@ -873,7 +739,7 @@ class LJ_API_Import {
}
}
- echo '' . __( "Your posts have all been imported, but wait - there's more! Now we need to process & import your comments." ) . '
';
+ echo '' . __( "Your posts have all been imported, but wait - there's more! Now we need to download & import your comments." ) . '
';
echo $this->next_step( 2, __( 'Download my comments »' ) );
$this->auto_submit();
}
@@ -884,143 +750,94 @@ class LJ_API_Import {
update_option( 'ljapi_step', 2 );
$this->username = get_option( 'ljapi_username' );
$this->password = get_option( 'ljapi_password' );
+ $this->ixr = new IXR_Client( $this->ixr_url );
+ echo '';
echo '
' . __( 'Downloading Comments' ) . '
';
- echo '
' . __( 'Now we will download your comments so we can process and import them...' ) . '
';
+ echo '
' . __( 'Now we will download your comments so we can import them (this could take a long time if you have lots of comments)...' ) . '
';
ob_flush(); flush();
- $result = $this->download_comments();
+ if ( !get_option( 'ljapi_usermap' ) ) {
+ // We haven't downloaded meta yet, so do that first
+ $result = $this->download_comment_meta();
+ if ( is_wp_error( $result ) )
+ return $result;
+ }
+
+ // Download a batch of actual comments
+ $result = $this->download_comment_bodies();
if ( is_wp_error( $result ) )
return $result;
- echo '
' . __( 'Your comments have all been downloaded to this server now, so we can process them and get them ready for importing.' ) . '
';
- echo $this->next_step( 3, __( 'Process my comment files »' ) );
- $this->auto_submit();
+ $maxid = get_option( 'ljapi_maxid' ) ? (int) get_option( 'ljapi_maxid' ) : 1;
+ $highest_id = (int) get_option( 'ljapi_highest_comment_id' );
+ if ( $maxid > $highest_id ) {
+ ?>
+
+ auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
+ ' . __( 'Your comments have all been imported now, but we still need to rebuild your conversation threads.' ) . '';
+ echo $this->next_step( 3, __( 'Rebuild my comment threads »' ) );
+ $this->auto_submit();
+ }
+ echo '
';
- echo '
' . __( 'Parsing Comments' ) . '
';
- echo '
' . __( 'Time to clean up your comments and get them into a format WordPress understands...' ) . '
';
- ob_flush(); flush();
-
- $files = get_option( 'ljapi_comment_xml_files' );
- if ( count( $files ) ) {
- $file = array_pop( $files );
-
- $result = $this->parse_comment_xml( $file );
- if ( is_wp_error( $result ) )
- return $result;
-
- update_option( 'ljapi_comment_xml_files', $files );
- }
-
- if ( count( $files ) ) {
- ?>
-
- auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
- ' . __( 'Yay, we finished processing all of your comment files! Now we need to re-build your conversation threads.' ) . '';
- echo $this->next_step( 4, __( 'Thread my comments »' ) );
- $this->auto_submit();
- }
- echo '
';
echo '
' . __( 'Threading Comments' ) . '
';
- echo '
' . __( 'Re-building your conversation threads ready for import...' ) . '
';
+ echo '
' . __( 'We are now re-building the threading of your comments (this can also take a while if you have lots of comments)...' ) . '
';
ob_flush(); flush();
- $files = get_option( 'ljapi_comment_cache_files' );
- if ( count( $files ) ) {
- $file = array_pop( $files );
-
- $result = $this->thread_comments( $file );
- if ( is_wp_error( $result ) )
- return $result;
-
- update_option( 'ljapi_comment_cache_files', $files );
+ // Only bother adding indexes if they have over 5000 comments (arbitrary number)
+ $imported_comments = $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->comments} WHERE comment_type = 'livejournal'" );
+ $added_indices = false;
+ if ( 5000 < $imported_comments ) {
+ include_once(ABSPATH . 'wp-admin/includes/upgrade.php');
+ $added_indices = true;
+ add_clean_index( $wpdb->comments, 'comment_type' );
+ add_clean_index( $wpdb->comments, 'comment_karma' );
+ add_clean_index( $wpdb->comments, 'comment_agent' );
}
- if ( count( $files ) ) {
- ?>
-
- auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
- ' . __( "Alrighty, your comments are all threaded. There's just one last step -- time to actually import them all now!" ) . '';
- echo '
' . __( 'This last part in particular can take a really long time if you have a lot of comments. You might want to go and do something else while you wait.' ) . '
';
- echo $this->next_step( 5, __( 'Import my threaded comments into WordPress »' ) );
- $this->auto_submit();
- }
- echo '
';
- echo '
' . __( 'Importing Comments' ) . '
';
- echo '
' . __( 'This is the big one -- we are now inserting your comment threads into WordPress...' ) . '
';
-
- $files = get_option( 'ljapi_comment_threaded_files' );
- echo '
' . sprintf( __( 'Importing cache file %d of %d' ), ( get_option( 'ljapi_comment_threaded_files_count' ) - count( $files ) + 1 ), get_option( 'ljapi_comment_threaded_files_count' ) ) . '
';
- ob_flush(); flush();
-
- if ( count( $files ) ) {
- $file = array_pop( $files );
-
- $result = $this->insert_comments( $file );
- if ( is_wp_error( $result ) )
- return $result;
-
- update_option( 'ljapi_comment_threaded_files', $files );
+ // Get LJ comments, which haven't been threaded yet, 5000 at a time and thread them
+ while ( $comments = $wpdb->get_results( "SELECT comment_ID, comment_agent FROM {$wpdb->comments} WHERE comment_type = 'livejournal' AND comment_agent != '0' LIMIT 5000", OBJECT ) ) {
+ foreach ( $comments as $comment ) {
+ $wpdb->update( $wpdb->comments,
+ array( 'comment_parent' => $this->get_wp_comment_ID( $comment->comment_agent ), 'comment_type' => 'livejournal-done' ),
+ array( 'comment_ID' => $comment->comment_ID ) );
+ }
+ wp_cache_flush();
+ $wpdb->flush();
}
- if ( count( $files ) ) {
- ?>
-
- auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
- cleanup();
- do_action( 'import_done', 'livejournal' );
- echo '
';
- printf( __( 'All done. Have fun!' ), get_option( 'home' ) );
- echo '
';
+ // Revert the comments table back to normal and optimize it to reclaim space
+ if ( $added_indices ) {
+ drop_index( $wpdb->comments, 'comment_type' );
+ drop_index( $wpdb->comments, 'comment_karma' );
+ drop_index( $wpdb->comments, 'comment_agent' );
+ $wpdb->query( "OPTIMIZE TABLE {$wpdb->comments}" );
}
+
+ // Clean up database and we're out
+ $this->cleanup();
+ do_action( 'import_done', 'livejournal' );
+ if ( $imported_comments > 1 )
+ echo '
' . sprintf( __( "Successfully re-threaded %d comments." ), number_format( $imported_comments ) ) . '
';
+ echo '
';
+ printf( __( 'All done. Have fun!' ), get_option( 'home' ) );
+ echo '
';
echo '