diff --git a/src/vs/base/parts/quickinput/browser/media/quickInput.css b/src/vs/base/parts/quickinput/browser/media/quickInput.css
index 45c205f92a524cf023fb00d1afa768a41cd305d2..fb4fb01293702caa5c921950610c145982670050 100644
--- a/src/vs/base/parts/quickinput/browser/media/quickInput.css
+++ b/src/vs/base/parts/quickinput/browser/media/quickInput.css
@@ -220,11 +220,20 @@
 
 .quick-input-list .quick-input-list-entry-action-bar {
 	display: flex;
-	visibility: hidden; /* not using display: none here to not flicker too much */
 	flex: 0;
 	overflow: visible;
 }
 
+.quick-input-list .quick-input-list-entry-action-bar .action-label {
+	/*
+	 * By default, actions in the quick input action bar are hidden
+	 * until hovered over them or selected. We do not use display:none
+	 * so that the amount of visual flickering is little by reserving the
+	 * space the button needs still.
+	 */
+	visibility: hidden;
+}
+
 .quick-input-list .quick-input-list-entry-action-bar .action-label.codicon {
 	margin: 0;
 	width: 19px;
@@ -244,8 +253,8 @@
 	margin-right: 4px; /* separate actions */
 }
 
-.quick-input-list .quick-input-list-entry.always-visible-actions .quick-input-list-entry-action-bar,
-.quick-input-list .quick-input-list-entry:hover .quick-input-list-entry-action-bar,
-.quick-input-list .monaco-list-row.focused .quick-input-list-entry-action-bar {
+.quick-input-list .quick-input-list-entry .quick-input-list-entry-action-bar .action-label.always-visible,
+.quick-input-list .quick-input-list-entry:hover .quick-input-list-entry-action-bar .action-label,
+.quick-input-list .monaco-list-row.focused .quick-input-list-entry-action-bar .action-label {
 	visibility: visible;
 }
diff --git a/src/vs/base/parts/quickinput/browser/quickInputList.ts b/src/vs/base/parts/quickinput/browser/quickInputList.ts
index d339c421185591052d2a182c93f9ae17f7cad818..9c72c887375dce1f64df38589c5976bc6f4b3c50 100644
--- a/src/vs/base/parts/quickinput/browser/quickInputList.ts
+++ b/src/vs/base/parts/quickinput/browser/quickInputList.ts
@@ -181,7 +181,11 @@ class ListElementRenderer implements IListRenderer<ListElement, IListElementTemp
 		const buttons = element.item.buttons;
 		if (buttons && buttons.length) {
 			data.actionBar.push(buttons.map((button, index) => {
-				const action = new Action(`id-${index}`, '', button.iconClass || (button.iconPath ? getIconClass(button.iconPath) : undefined), true, () => {
+				let cssClasses = button.iconClass || (button.iconPath ? getIconClass(button.iconPath) : undefined);
+				if (button.alwaysVisible) {
+					cssClasses = cssClasses ? `${cssClasses} always-visible` : 'always-visible';
+				}
+				const action = new Action(`id-${index}`, '', cssClasses, true, () => {
 					element.fireButtonTriggered({
 						button,
 						item: element.item
@@ -195,12 +199,6 @@ class ListElementRenderer implements IListRenderer<ListElement, IListElementTemp
 		} else {
 			dom.removeClass(data.entry, 'has-actions');
 		}
-
-		if (element.item.buttonsAlwaysVisible) {
-			dom.addClass(data.entry, 'always-visible-actions');
-		} else {
-			dom.removeClass(data.entry, 'always-visible-actions');
-		}
 	}
 
 	disposeElement(element: ListElement, index: number, data: IListElementTemplateData): void {
diff --git a/src/vs/base/parts/quickinput/common/quickInput.ts b/src/vs/base/parts/quickinput/common/quickInput.ts
index 8ba8467533b27283b9b4bfe09183c7c6c8776f85..d71b82cdbed4ef056e52420475e33e1dde903371 100644
--- a/src/vs/base/parts/quickinput/common/quickInput.ts
+++ b/src/vs/base/parts/quickinput/common/quickInput.ts
@@ -35,11 +35,6 @@ export interface IQuickPickItem {
 	strikethrough?: boolean;
 	highlights?: IQuickPickItemHighlights;
 	buttons?: IQuickInputButton[];
-	/**
-	 * Wether to always show the buttons. By default buttons
-	 * are only visible when hovering over them with the mouse
-	 */
-	buttonsAlwaysVisible?: boolean;
 	picked?: boolean;
 	alwaysShow?: boolean;
 }
@@ -290,6 +285,11 @@ export interface IQuickInputButton {
 	/** iconPath or iconClass required */
 	iconClass?: string;
 	tooltip?: string;
+	/**
+	 * Wether to always show the button. By default buttons
+	 * are only visible when hovering over them with the mouse
+	 */
+	alwaysVisible?: boolean;
 }
 
 export interface IQuickPickItemButtonEvent<T extends IQuickPickItem> {
diff --git a/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts b/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts
index 8e60b2d12bd9e0e22e63abb0fc13f084103ad4ff..7d30370b0d3d932874a723049a4ee348d9640866 100644
--- a/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts
+++ b/src/vs/workbench/browser/parts/editor/editorQuickAccess.ts
@@ -99,11 +99,11 @@ export abstract class BaseEditorQuickAccessProvider extends PickerQuickAccessPro
 				description: editor.getDescription(),
 				iconClasses: getIconClasses(this.modelService, this.modeService, resource),
 				italic: !this.editorGroupService.getGroup(groupId)?.isPinned(editor),
-				buttonsAlwaysVisible: isDirty,
 				buttons: [
 					{
 						iconClass: isDirty ? 'dirty-editor codicon-circle-filled' : 'codicon-close',
-						tooltip: localize('closeEditor', "Close Editor")
+						tooltip: localize('closeEditor', "Close Editor"),
+						alwaysVisible: isDirty
 					}
 				],
 				trigger: async () => {
diff --git a/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts b/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts
index 7eeebaa61844c9b9bf25aed453bed1e864fbe2ee..1b32475bf143f3476958c96e8d898a44510bc5b1 100644
--- a/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts
+++ b/src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts
@@ -404,7 +404,6 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider<IAnyt
 				localize('filePickAriaLabel', "{0}, file picker", label),
 			description,
 			iconClasses: getIconClasses(this.modelService, this.modeService, resource),
-			buttonsAlwaysVisible: isDirty,
 			buttons: (() => {
 				const openSideBySideDirection = this.configuration.openSideBySideDirection;
 				const buttons: IQuickInputButton[] = [];
@@ -419,15 +418,8 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider<IAnyt
 				if (isEditorHistoryEntry) {
 					buttons.push({
 						iconClass: isDirty ? 'dirty-anything codicon-circle-filled' : 'codicon-close',
-						tooltip: localize('closeEditor', "Remove from Recently Opened")
-					});
-				}
-
-				// Dirty indicator
-				else if (isDirty) {
-					buttons.push({
-						iconClass: 'codicon-circle-filled',
-						tooltip: localize('dirtyFile', "Dirty File")
+						tooltip: localize('closeEditor', "Remove from Recently Opened"),
+						alwaysVisible: isDirty
 					});
 				}
 
@@ -441,7 +433,7 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider<IAnyt
 						this.openAnything(resourceOrEditor, { keyMods, range, forceOpenSideBySide: true });
 						return TriggerAction.CLOSE_PICKER;
 
-					// Remove from History / Dirty Indicator
+					// Remove from History
 					case 1:
 						if (!URI.isUri(resourceOrEditor)) {
 							this.historyService.remove(resourceOrEditor);