Added some improvements to changes made in [4143] for #1854.

src/core.js

@@ -29,6 +29,9 @@ window.$ = jQuery;
 // (both of which we optimize for)
 var quickExpr = /^[^<]*(<(.|\s)+>)[^>]*$|^#(\w+)$/;
 
+// Is it a simple selector
+var isSimple = /^.[^:#\[\.]*$/;
+
 jQuery.fn = jQuery.prototype = {
 	init: function( selector, context ) {
 		// Make sure that a selection was provided
@@ -342,14 +345,16 @@ jQuery.fn = jQuery.prototype = {
 	},
 
 	not: function( selector ) {
-		if (selector.constructor == String)
+		if ( selector.constructor == String )
 			// test special case where just one selector is passed in
-			if ( /^.[^:#\[\.]*$/.test(selector) )
+			if ( isSimple.test( selector ) )
 				return this.pushStack( jQuery.multiFilter( selector, this, true ) );
 			else
 				selector = jQuery.multiFilter( selector, this );
 
-		return this.pushStack( jQuery.removeFromArray( selector, this ) );
+		return this.filter(function() {
+			return jQuery.inArray( this, selector ) < 0;
+		});
 	},
 
 	add: function( selector ) {
@@ -1092,13 +1097,6 @@ jQuery.extend({
 		return -1;
 	},
 
-	removeFromArray: function( remove, from ) {
-		var isArrayLike = remove.length && remove[remove.length - 1] !== undefined;
-		return jQuery.grep(from, function(elem) {
-				return isArrayLike ? jQuery.inArray( elem, remove ) < 0 : elem != from;
-			});
-	},
-
 	merge: function( first, second ) {
 		// We have to loop this way because IE & Opera overwrite the length
 		// expando of getElementsByTagName

src/selector.js

@@ -321,10 +321,9 @@ jQuery.extend({
 			// keeping it out of the expression list
 			if ( m[1] == ":" && m[2] == "not" )
 				// optimize if only one selector found (most common case)
-				if ( /^.[^:#\[\.]*$/.test(m[3]) )
-					r = jQuery.filter(m[3], r, true).r;
-				else
-					r = jQuery.removeFromArray(jQuery.multiFilter(m[3], r), r);
+				r = isSimple.test( m[3] ) ?
+					jQuery.filter(m[3], r, true).r :
+					jQuery( r ).not( m[3] );
 
 			// We can get a big speed boost by filtering by class here
 			else if ( m[1] == "." )