diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c
index 50c8fece1681ca783cfcc2bfc4c0e8e1313d2bf0..9b9d12bbdc1d8efb050cd862253e4fc358f3e265 100644
--- a/tools/perf/util/hist.c
+++ b/tools/perf/util/hist.c
@@ -100,6 +100,8 @@ static void hist_entry__decay(struct hist_entry *he)
 
 static bool hists__decay_entry(struct hists *hists, struct hist_entry *he)
 {
+	if (he->period == 0)
+		return true;
 	hists->stats.total_period -= he->period;
 	hist_entry__decay(he);
 	hists->stats.total_period += he->period;
@@ -114,8 +116,12 @@ void hists__decay_entries(struct hists *hists)
 	while (next) {
 		n = rb_entry(next, struct hist_entry, rb_node);
 		next = rb_next(&n->rb_node);
-
-		if (hists__decay_entry(hists, n)) {
+		/*
+		 * We may be annotating this, for instance, so keep it here in
+		 * case some it gets new samples, we'll eventually free it when
+		 * the user stops browsing and it agains gets fully decayed.
+		 */
+		if (hists__decay_entry(hists, n) && !n->used) {
 			rb_erase(&n->rb_node, &hists->entries);
 
 			if (sort__need_collapse)
diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h
index 03851e301721f0f8babfa5883bc8000f1c157051..3f67ae3957521aabed2bb1e2bc5931aa57f4930b 100644
--- a/tools/perf/util/sort.h
+++ b/tools/perf/util/sort.h
@@ -64,6 +64,7 @@ struct hist_entry {
 
 	bool			init_have_children;
 	char			level;
+	bool			used;
 	u8			filtered;
 	struct symbol		*parent;
 	union {
diff --git a/tools/perf/util/ui/browsers/hists.c b/tools/perf/util/ui/browsers/hists.c
index 9ece8435353865e6d6b262e6d24a527821022a99..fdc3c90696dcfb911fd3707ec44caa9f776d6f85 100644
--- a/tools/perf/util/ui/browsers/hists.c
+++ b/tools/perf/util/ui/browsers/hists.c
@@ -976,9 +976,13 @@ static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events,
 			he = hist_browser__selected_entry(browser);
 			if (he == NULL)
 				continue;
-
+			/*
+			 * Don't let this be freed, say, by hists__decay_entry.
+			 */
+			he->used = true;
 			hist_entry__tui_annotate(he, evsel->idx, nr_events,
 						 timer, arg, delay_secs);
+			he->used = false;
 			ui_browser__update_nr_entries(&browser->b, browser->hists->nr_entries);
 		} else if (choice == browse_map)
 			map__browse(browser->selection->map);