diff --git a/src/wp-includes/css/media-views.css b/src/wp-includes/css/media-views.css
index 859dbcc8b3..b8f53d01cd 100644
--- a/src/wp-includes/css/media-views.css
+++ b/src/wp-includes/css/media-views.css
@@ -925,7 +925,7 @@
 }
 
 .attachment .close {
-	display: none;
+	display: block;
 	position: absolute;
 	top: 5px;
 	right: 5px;
@@ -944,15 +944,16 @@
 	border-radius: 3px;
 	-webkit-box-shadow: 0 0 0 1px rgba( 0, 0, 0, 0.3 );
 	box-shadow: 0 0 0 1px rgba( 0, 0, 0, 0.3 );
+	transition-duration: none;
+	transition-property: none;
 }
 
-.attachment .close:hover {
+.attachment a.close:hover,
+.attachment a.close:focus {
 	-webkit-box-shadow: 0 0 0 1px rgba( 0, 0, 0, 0.6 );
 	box-shadow: 0 0 0 1px rgba( 0, 0, 0, 0.6 );
-}
+	background-position: -36px 4px;
 
-.attachment:hover .close {
-	display: block;
 }
 
 .attachment .check {
diff --git a/src/wp-includes/js/media-views.js b/src/wp-includes/js/media-views.js
index ba06ca2b76..cc0f987fb7 100644
--- a/src/wp-includes/js/media-views.js
+++ b/src/wp-includes/js/media-views.js
@@ -3016,6 +3016,9 @@
 
 			// Browse our library of attachments.
 			this.content.set( view );
+
+			// Trigger the controller to set focus
+			view.controller.trigger( 'edit:selection', this );
 		},
 
 		editImageContent: function() {
@@ -5096,6 +5099,7 @@
 			'click .close':                   'removeFromLibrary',
 			'click .check':                   'checkClickHandler',
 			'click a':                        'preventDefault',
+			'keydown .close':                 'removeFromLibrary',
 			'keydown':                        'toggleSelectionHandler'
 		},
 
@@ -5537,6 +5541,11 @@
 		 * @param {Object} event
 		 */
 		removeFromLibrary: function( event ) {
+			// Catch enter and space events
+			if ( 'keydown' === event.type && 13 !== event.keyCode && 32 !== event.keyCode ) {
+				return;
+			}
+
 			// Stop propagation so the model isn't selected.
 			event.stopPropagation();
 
@@ -6236,7 +6245,7 @@
 			});
 
 			this.listenTo( this.controller, 'toggle:upload:attachment', _.bind( this.toggleUploader, this ) );
-
+			this.controller.on( 'edit:selection', this.editSelection );
 			this.createToolbar();
 			if ( this.options.sidebar ) {
 				this.createSidebar();
@@ -6255,6 +6264,11 @@
 
 			this.collection.on( 'add remove reset', this.updateContent, this );
 		},
+
+		editSelection: function( modal ) {
+			modal.$el.find( '.media-button-backToLibrary' ).focus();
+		},
+
 		/**
 		 * @returns {wp.media.view.AttachmentsBrowser} Returns itself to allow chaining
 		 */