diff --git a/src/wp-admin/includes/taxonomy.php b/src/wp-admin/includes/taxonomy.php
index fdeec03c90..524cf05d27 100644
--- a/src/wp-admin/includes/taxonomy.php
+++ b/src/wp-admin/includes/taxonomy.php
@@ -242,7 +242,7 @@ function get_terms_to_edit( $post_id, $taxonomy = 'post_tag' ) {
 	$terms = get_object_term_cache( $post_id, $taxonomy );
 	if ( false === $terms ) {
 		$terms = wp_get_object_terms( $post_id, $taxonomy );
-		wp_cache_add( $post_id, $terms, $taxonomy . '_relationships' );
+		wp_cache_add( $post_id, wp_list_pluck( $terms, 'term_id' ), $taxonomy . '_relationships' );
 	}
 
 	if ( ! $terms ) {
diff --git a/src/wp-admin/includes/template.php b/src/wp-admin/includes/template.php
index 476389a121..42833231a6 100644
--- a/src/wp-admin/includes/template.php
+++ b/src/wp-admin/includes/template.php
@@ -311,7 +311,7 @@ function get_inline_data($post) {
 			$terms = get_object_term_cache( $post->ID, $taxonomy_name );
 			if ( false === $terms ) {
 				$terms = wp_get_object_terms( $post->ID, $taxonomy_name );
-				wp_cache_add( $post->ID, $terms, $taxonomy_name . '_relationships' );
+				wp_cache_add( $post->ID, wp_list_pluck( $terms, 'term_id' ), $taxonomy_name . '_relationships' );
 			}
 			$term_ids = empty( $terms ) ? array() : wp_list_pluck( $terms, 'term_id' );
 
diff --git a/src/wp-includes/category-template.php b/src/wp-includes/category-template.php
index 7c03a076dc..3b66d0f0fb 100644
--- a/src/wp-includes/category-template.php
+++ b/src/wp-includes/category-template.php
@@ -1187,18 +1187,11 @@ function get_the_terms( $post, $taxonomy ) {
 	if ( false === $terms ) {
 		$terms = wp_get_object_terms( $post->ID, $taxonomy );
 		if ( ! is_wp_error( $terms ) ) {
-			$to_cache = array();
-			foreach ( $terms as $key => $term ) {
-				$to_cache[ $key ] = $term->data;
-			}
-			wp_cache_add( $post->ID, $to_cache, $taxonomy . '_relationships' );
+			$term_ids = wp_list_pluck( $terms, 'term_id' );
+			wp_cache_add( $post->ID, $term_ids, $taxonomy . '_relationships' );
 		}
 	}
 
-	if ( ! is_wp_error( $terms ) ) {
-		$terms = array_map( 'get_term', $terms );
-	}
-
 	/**
 	 * Filters the list of terms attached to the given post.
 	 *
diff --git a/src/wp-includes/taxonomy.php b/src/wp-includes/taxonomy.php
index 68570f99de..8a6dae651a 100644
--- a/src/wp-includes/taxonomy.php
+++ b/src/wp-includes/taxonomy.php
@@ -3239,15 +3239,42 @@ function clean_term_cache($ids, $taxonomy = '', $clean_taxonomy = true) {
 /**
  * Retrieves the taxonomy relationship to the term object id.
  *
+ * Upstream functions (like `get_the_terms()` and `is_object_in_term()`) are responsible for populating the
+ * object-term relationship cache. The current function only fetches relationship data that is already in the cache.
+ *
  * @since 2.3.0
  *
  * @param int    $id       Term object ID.
  * @param string $taxonomy Taxonomy name.
- * @return bool|mixed Empty array if $terms found, but not `$taxonomy`. False if nothing is in cache
- *                    for `$taxonomy` and `$id`.
+ * @return bool|array Array of `WP_Term` objects, if cached False if cache is empty for `$taxonomy` and `$id`.
  */
 function get_object_term_cache( $id, $taxonomy ) {
-	return wp_cache_get( $id, "{$taxonomy}_relationships" );
+	$_term_ids = wp_cache_get( $id, "{$taxonomy}_relationships" );
+
+	// We leave the priming of relationship caches to upstream functions.
+	if ( false === $_term_ids ) {
+		return false;
+	}
+
+	// Backward compatibility for if a plugin is putting objects into the cache, rather than IDs.
+	$term_ids = array();
+	foreach ( $_term_ids as $term_id ) {
+		if ( is_numeric( $term_id ) ) {
+			$term_ids[] = intval( $term_id );
+		} elseif ( isset( $term_id->term_id ) ) {
+			$term_ids[] = intval( $term_id->term_id );
+		}
+	}
+
+	// Fill the term objects.
+	_prime_term_caches( $term_ids );
+
+	$terms = array();
+	foreach ( $term_ids as $term_id ) {
+		$terms[] = wp_cache_get( $term_id, 'terms' );
+	}
+
+	return array_map( 'get_term', $terms );
 }
 
 /**
@@ -3297,8 +3324,9 @@ function update_object_term_cache($object_ids, $object_type) {
 	) );
 
 	$object_terms = array();
-	foreach ( (array) $terms as $term )
-		$object_terms[$term->object_id][$term->taxonomy][] = $term;
+	foreach ( (array) $terms as $term ) {
+		$object_terms[ $term->object_id ][ $term->taxonomy ][] = $term->term_id;
+	}
 
 	foreach ( $ids as $id ) {
 		foreach ( $taxonomies as $taxonomy ) {
@@ -3504,6 +3532,32 @@ function _pad_term_counts( &$terms, $taxonomy ) {
 			$terms_by_id[$id]->count = count($items);
 }
 
+/**
+ * Adds any terms from the given IDs to the cache that do not already exist in cache.
+ *
+ * @since 4.6.0
+ * @access private
+ *
+ * @global wpdb $wpdb WordPress database abstraction object.
+ *
+ * @param array $term_ids          Array of term IDs.
+ * @param bool  $update_meta_cache Optional. Whether to update the meta cache. Default true.
+ */
+function _prime_term_caches( $term_ids, $update_meta_cache = true ) {
+	global $wpdb;
+
+	$non_cached_ids = _get_non_cached_ids( $term_ids, 'terms' );
+	if ( ! empty( $non_cached_ids ) ) {
+		$fresh_terms = $wpdb->get_results( sprintf( "SELECT t.*, tt.* FROM $wpdb->terms AS t INNER JOIN $wpdb->term_taxonomy AS tt ON t.term_id = tt.term_id WHERE t.term_id IN (%s)", join( ",", array_map( 'intval', $non_cached_ids ) ) ) );
+
+		update_term_cache( $fresh_terms, $update_meta_cache );
+
+		if ( $update_meta_cache ) {
+			update_termmeta_cache( $non_cached_ids );
+		}
+	}
+}
+
 //
 // Default callbacks
 //
@@ -4211,7 +4265,7 @@ function is_object_in_term( $object_id, $taxonomy, $terms = null ) {
 	$object_terms = get_object_term_cache( $object_id, $taxonomy );
 	if ( false === $object_terms ) {
 		$object_terms = wp_get_object_terms( $object_id, $taxonomy, array( 'update_term_meta_cache' => false ) );
-		wp_cache_set( $object_id, $object_terms, "{$taxonomy}_relationships" );
+		wp_cache_set( $object_id, wp_list_pluck( $object_terms, 'term_id' ), "{$taxonomy}_relationships" );
 	}
 
 	if ( is_wp_error( $object_terms ) )
diff --git a/tests/phpunit/tests/term/getTheTerms.php b/tests/phpunit/tests/term/getTheTerms.php
index d7bb7fc42d..a9bcf783cc 100644
--- a/tests/phpunit/tests/term/getTheTerms.php
+++ b/tests/phpunit/tests/term/getTheTerms.php
@@ -75,25 +75,6 @@ class Tests_Term_GetTheTerms extends WP_UnitTestCase {
 		$this->assertEquals( 'This description is even more amazing!', $terms[0]->description );
 	}
 
-	/**
-	 * @ticket 34262
-	 */
-	public function test_get_the_terms_should_not_cache_wp_term_objects() {
-		$p = self::$post_ids[0];
-		register_taxonomy( 'wptests_tax', 'post' );
-		$t = self::factory()->term->create( array( 'taxonomy' => 'wptests_tax' ) );
-		wp_set_object_terms( $p, $t, 'wptests_tax' );
-
-		// Prime the cache.
-		$terms = get_the_terms( $p, 'wptests_tax' );
-
-		$cached = get_object_term_cache( $p, 'wptests_tax' );
-
-		$this->assertNotEmpty( $cached );
-		$this->assertSame( $t, (int) $cached[0]->term_id );
-		$this->assertNotInstanceOf( 'WP_Term', $cached[0] );
-	}
-
 	/**
 	 * @ticket 34262
 	 */
@@ -170,4 +151,50 @@ class Tests_Term_GetTheTerms extends WP_UnitTestCase {
 		$terms = get_the_terms( $p, 'this-taxonomy-does-not-exist' );
 		$this->assertTrue( is_wp_error( $terms ) );
 	}
+
+	/**
+	 * @ticket 36814
+	 */
+	public function test_count_should_not_be_improperly_cached() {
+		register_taxonomy( 'wptests_tax', 'post' );
+
+		$t = self::factory()->term->create( array( 'taxonomy' => 'wptests_tax' ) );
+
+		wp_set_object_terms( self::$post_ids[0], $t, 'wptests_tax' );
+
+		$terms = get_the_terms( self::$post_ids[0], 'wptests_tax' );
+		$this->assertSame( 1, $terms[0]->count );
+
+		wp_set_object_terms( self::$post_ids[1], $t, 'wptests_tax' );
+
+		$terms = get_the_terms( self::$post_ids[0], 'wptests_tax' );
+		$this->assertSame( 2, $terms[0]->count );
+	}
+
+	/**
+	 * @ticket 36814
+	 */
+	public function test_uncached_terms_should_be_primed_with_a_single_query() {
+		global $wpdb;
+
+		register_taxonomy( 'wptests_tax', 'post' );
+
+		$terms = self::factory()->term->create_many( 3, array( 'taxonomy' => 'wptests_tax' ) );
+
+		wp_set_object_terms( self::$post_ids[0], $terms, 'wptests_tax' );
+
+		get_the_terms( self::$post_ids[0], 'wptests_tax' );
+
+		// Clean cache for two of the terms.
+		clean_term_cache( array( $terms[0], $terms[1] ), 'wptests_tax', false );
+
+		$num_queries = $wpdb->num_queries;
+		$found = get_the_terms( self::$post_ids[0], 'wptests_tax' );
+
+		$this->assertEqualSets( $terms, wp_list_pluck( $found, 'term_id' ) );
+
+		$num_queries++;
+		$this->assertSame( $num_queries, $wpdb->num_queries );
+
+	}
 }