diff --git a/Gruntfile.js b/Gruntfile.js
index fd048b1cca..906900dac0 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -192,6 +192,28 @@ module.exports = function(grunt) {
 						}
 					]
 				},
+				properties : [
+					{
+						name: 'swap-dashicons-left-right-arrows',
+						expr: /content/im,
+						action: function( prop, value ) {
+							if ( value === '"\\f141"' ) { // dashicons-arrow-left
+								value = '"\\f139"';
+							} else if ( value === '"\\f340"' ) { // dashicons-arrow-left-alt
+								value = '"\\f344"';
+							} else if ( value === '"\\f341"' ) { // dashicons-arrow-left-alt2
+								value = '"\\f345"';
+							} else if ( value === '"\\f139"' ) { // dashicons-arrow-right
+								value = '"\\f141"';
+							} else if ( value === '"\\f344"' ) { // dashicons-arrow-right-alt
+								value = '"\\f340"';
+							} else if ( value === '"\\f345"' ) { // dashicons-arrow-right-alt2
+								value = '"\\f341"';
+							}
+							return { prop: prop, value: value };
+						}
+					}
+				],
 				saveUnmodified: false
 			},
 			core: {
@@ -201,7 +223,10 @@ module.exports = function(grunt) {
 				ext: '-rtl.css',
 				src: [
 					'wp-admin/css/*.css',
-					'wp-includes/css/*.css'
+					'wp-includes/css/*.css',
+
+					// Exceptions
+					'!wp-includes/css/dashicons.css'
 				]
 			},
 			colors: {
diff --git a/src/wp-admin/css/customize-controls.css b/src/wp-admin/css/customize-controls.css
index 82713f4dc2..5ff93ad857 100644
--- a/src/wp-admin/css/customize-controls.css
+++ b/src/wp-admin/css/customize-controls.css
@@ -162,10 +162,6 @@ body {
 	z-index: 0;
 }
 
-.rtl .control-section.control-panel > .accordion-section-title:after {
-	content: "\f341";
-}
-
 #customize-theme-controls .control-section.control-panel > h3.accordion-section-title:focus:after,
 #customize-theme-controls .control-section.control-panel > h3.accordion-section-title:hover:after {
 	background: #ddd;
@@ -262,10 +258,6 @@ body {
 	left: 13px;
 }
 
-.rtl .control-panel-back:before {
-	content: "\f345";
-}
-
 .in-sub-panel .control-panel-back {
 	left: 0;
 }
@@ -940,10 +932,6 @@ p.customize-section-description {
 	content: "\f345";
 }
 
-.rtl .control-panel-back.themes-panel-back:before {
-	content: "\f341";
-}
-
 /* Details View */
 .wp-customizer .theme-overlay {
 	display: none;
@@ -988,7 +976,7 @@ p.customize-section-description {
 		right: 0;
 		top: 0;
 		bottom: 0;
-	}	
+	}
 }
 
 
diff --git a/src/wp-admin/css/themes.css b/src/wp-admin/css/themes.css
index 1b9f276536..541c8eb954 100644
--- a/src/wp-admin/css/themes.css
+++ b/src/wp-admin/css/themes.css
@@ -499,14 +499,12 @@
 	font-weight: 300;
 }
 
-.theme-overlay .theme-header .left:before,
-.rtl .theme-overlay .theme-header .right:before {
-	content: '\f341';
+.theme-overlay .theme-header .left:before {
+	content: "\f341";
 }
 
-.theme-overlay .theme-header .right:before,
-.rtl .theme-overlay .theme-header .left:before {
-	content: '\f345';
+.theme-overlay .theme-header .right:before {
+	content: "\f345";
 }
 
 
@@ -1462,14 +1460,6 @@ body.full-overlay-active {
 	pointer-events: none;
 }
 
-.rtl .theme-install-overlay .previous-theme:before {
-	content: "\f345";
-}
-
-.rtl .theme-install-overlay .next-theme:before {
-	content: "\f341";
-}
-
 /* Collapse Button */
 .wp-full-overlay a.collapse-sidebar {
 	position: absolute;