diff --git a/tools/perf/util/map.c b/tools/perf/util/map.c
index eec9b282c04790fb4115c3435822547ce04f84be..c9ba49566981ea9d9cdb0a8401404644e9d9b816 100644
--- a/tools/perf/util/map.c
+++ b/tools/perf/util/map.c
@@ -288,7 +288,7 @@ bool map__has_symbols(const struct map *map)
 
 static void map__exit(struct map *map)
 {
-	BUG_ON(!RB_EMPTY_NODE(&map->rb_node));
+	BUG_ON(refcount_read(&map->refcnt) != 0);
 	dso__zput(map->dso);
 }