From 2580a9c7f11c7fe269bec27aa90e2ee57e319d2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= <xavier@metamarkets.com>
Date: Wed, 26 Mar 2014 16:49:39 -0700
Subject: [PATCH] mark long-running tests as @Ignore + docs

---
 .../hyperloglog/HyperLogLogCollectorTest.java | 23 +++++++++++++++----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollectorTest.java b/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollectorTest.java
index 8968c9a867..b13ad5690c 100644
--- a/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollectorTest.java
+++ b/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollectorTest.java
@@ -22,6 +22,7 @@ package io.druid.query.aggregation.hyperloglog;
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.nio.ByteBuffer;
@@ -70,17 +71,29 @@ public class HyperLogLogCollectorTest
     }
   }
 
-  //  @Test
+
+  /**
+   * This is a very long running test, disabled by default.
+   * It is meant to catch issues when combining a large numer of HLL objects.
+   *
+   * It compares adding all the values to one HLL vs.
+   * splitting up values into HLLs of 100 values each, and folding those HLLs into a single main HLL.
+   *
+   * When reaching very large cardinalities (>> 50,000,000), offsets are mismatched between the main HLL and the ones
+   * with 100 values, requiring  a floating max as described in
+   * http://druid.io/blog/2014/02/18/hyperloglog-optimizations-for-real-world-systems.html
+   */
+  @Ignore @Test
   public void testHighCardinalityRollingFold() throws Exception
   {
     final HyperLogLogCollector rolling = HyperLogLogCollector.makeLatestCollector();
     final HyperLogLogCollector simple = HyperLogLogCollector.makeLatestCollector();
 
-    int count;
     MessageDigest md = MessageDigest.getInstance("SHA-1");
     HyperLogLogCollector tmp = HyperLogLogCollector.makeLatestCollector();
 
-    for (count = 0; count < 5000000; ++count) {
+    int count;
+    for (count = 0; count < 100_000_000; ++count) {
       md.update(Integer.toString(count).getBytes());
 
       byte[] hashed = fn.hashBytes(md.digest()).asBytes();
@@ -110,14 +123,14 @@ public class HyperLogLogCollectorTest
     Assert.assertEquals(n, rolling.estimateCardinality(), n * 0.05);
   }
 
-  //@Test
+  @Ignore @Test
   public void testHighCardinalityRollingFold2() throws Exception
   {
     final HyperLogLogCollector rolling = HyperLogLogCollector.makeLatestCollector();
     int count;
     long start = System.currentTimeMillis();
 
-    for (count = 0; count < 5000000; ++count) {
+    for (count = 0; count < 50_000_000; ++count) {
       HyperLogLogCollector theCollector = HyperLogLogCollector.makeLatestCollector();
       theCollector.add(fn.hashLong(count).asBytes());
       rolling.fold(theCollector);
-- 
GitLab