working indexer console

merger/src/main/java/com/metamx/druid/merger/common/task/AppendTask.java

@@ -35,8 +35,6 @@ import com.metamx.druid.index.v1.IndexableAdapter;
 import com.metamx.druid.index.v1.QueryableIndexIndexableAdapter;
 import com.metamx.druid.index.v1.Rowboat;
 import com.metamx.druid.index.v1.RowboatFilteringIndexAdapter;
-
-
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;

merger/src/main/java/com/metamx/druid/merger/common/task/DefaultMergeTask.java

@@ -31,8 +31,6 @@ import com.metamx.druid.index.QueryableIndex;
 import com.metamx.druid.index.v1.IndexIO;
 import com.metamx.druid.index.v1.IndexMerger;
 
-
-
 import javax.annotation.Nullable;
 import java.io.File;
 import java.util.List;

merger/src/main/java/com/metamx/druid/merger/common/task/IndexTask.java

@@ -33,8 +33,6 @@ import com.metamx.druid.merger.common.actions.SpawnTasksAction;
 import com.metamx.druid.realtime.FirehoseFactory;
 import com.metamx.druid.realtime.Schema;
 import com.metamx.druid.shard.NoneShardSpec;
-
-
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 

merger/src/main/java/com/metamx/druid/merger/common/task/KillTask.java

 package com.metamx.druid.merger.common.task;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.metamx.common.ISE;
 import com.metamx.common.logger.Logger;
 import com.metamx.druid.client.DataSegment;
+import com.metamx.druid.merger.common.TaskLock;
 import com.metamx.druid.merger.common.TaskStatus;
 import com.metamx.druid.merger.common.TaskToolbox;
 import com.metamx.druid.merger.common.actions.LockListAction;
 import com.metamx.druid.merger.common.actions.SegmentListUnusedAction;
 import com.metamx.druid.merger.common.actions.SegmentNukeAction;
-import com.metamx.druid.merger.common.TaskLock;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 

merger/src/main/java/com/metamx/druid/merger/coordinator/TaskMasterLifecycle.java

@@ -100,7 +100,7 @@ public class TaskMasterLifecycle
           leaderLifecycle.addManagedInstance(taskRunner);
           Initialization.makeServiceDiscoveryClient(curator, serviceDiscoveryConfig, leaderLifecycle);
           leaderLifecycle.addManagedInstance(taskConsumer);
-          leaderLifecycle.addManagedInstance(scheduler);
+          leaderLifecycle.addManagedInstance(resourceManagementScheduler);
 
           leading = true;
 

merger/src/main/java/com/metamx/druid/merger/coordinator/TaskRunnerWorkItem.java

@@ -19,6 +19,7 @@
 
 package com.metamx.druid.merger.coordinator;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.metamx.druid.merger.common.TaskCallback;
 import com.metamx.druid.merger.common.task.Task;
 import org.joda.time.DateTime;
@@ -49,6 +50,7 @@ public class TaskRunnerWorkItem implements Comparable<TaskRunnerWorkItem>
     this.createdTime = createdTime;
   }
 
+  @JsonProperty
   public Task getTask()
   {
     return task;
@@ -64,11 +66,13 @@ public class TaskRunnerWorkItem implements Comparable<TaskRunnerWorkItem>
     return retryPolicy;
   }
 
+  @JsonProperty
   public DateTime getCreatedTime()
   {
     return createdTime;
   }
 
+  @JsonProperty
   public DateTime getQueueInsertionTime()
   {
     return queueInsertionTime;

merger/src/main/java/com/metamx/druid/merger/coordinator/ZkWorker.java

@@ -20,6 +20,7 @@
 package com.metamx.druid.merger.coordinator;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonUnwrapped;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;

merger/src/main/java/com/metamx/druid/merger/coordinator/http/IndexerCoordinatorNode.java

@@ -41,11 +41,10 @@ import com.metamx.common.logger.Logger;
 import com.metamx.druid.RegisteringNode;
 import com.metamx.druid.db.DbConnector;
 import com.metamx.druid.db.DbConnectorConfig;
-import com.metamx.druid.http.ComputeMain;
 import com.metamx.druid.http.GuiceServletConfig;
+import com.metamx.druid.http.MasterMain;
 import com.metamx.druid.http.RedirectFilter;
 import com.metamx.druid.http.RedirectInfo;
-import com.metamx.druid.http.RedirectServlet;
 import com.metamx.druid.http.StatusServlet;
 import com.metamx.druid.initialization.Initialization;
 import com.metamx.druid.initialization.ServerConfig;
@@ -113,6 +112,7 @@ import org.mortbay.jetty.servlet.Context;
 import org.mortbay.jetty.servlet.DefaultServlet;
 import org.mortbay.jetty.servlet.FilterHolder;
 import org.mortbay.jetty.servlet.ServletHolder;
+import org.mortbay.resource.ResourceCollection;
 import org.skife.config.ConfigurationObjectFactory;
 import org.skife.jdbi.v2.DBI;
 
@@ -286,7 +286,11 @@ public class IndexerCoordinatorNode extends RegisteringNode
     final Context staticContext = new Context(server, "/static", Context.SESSIONS);
     staticContext.addServlet(new ServletHolder(new DefaultServlet()), "/*");
 
-    staticContext.setResourceBase(IndexerCoordinatorNode.class.getClassLoader().getResource("static").toExternalForm());
+    ResourceCollection resourceCollection = new ResourceCollection(new String[] {
+        IndexerCoordinatorNode.class.getClassLoader().getResource("static").toExternalForm(),
+        IndexerCoordinatorNode.class.getClassLoader().getResource("indexer_static").toExternalForm()
+    });
+    staticContext.setBaseResource(resourceCollection);
 
     final Context root = new Context(server, "/", Context.SESSIONS);
     root.addServlet(new ServletHolder(new StatusServlet()), "/status");

merger/src/main/java/com/metamx/druid/merger/coordinator/http/IndexerCoordinatorResource.java

@@ -21,6 +21,7 @@ package com.metamx.druid.merger.coordinator.http;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.inject.Inject;
 import com.metamx.common.logger.Logger;
@@ -31,6 +32,8 @@ import com.metamx.druid.merger.common.task.Task;
 import com.metamx.druid.merger.coordinator.TaskMasterLifecycle;
 import com.metamx.druid.merger.coordinator.TaskStorageQueryAdapter;
 import com.metamx.druid.merger.coordinator.config.IndexerCoordinatorConfig;
+import com.metamx.druid.merger.coordinator.scaling.AutoScalingData;
+import com.metamx.druid.merger.coordinator.scaling.ScalingStats;
 import com.metamx.druid.merger.coordinator.setup.WorkerSetupData;
 import com.metamx.druid.merger.coordinator.setup.WorkerSetupManager;
 import com.metamx.emitter.service.ServiceEmitter;
@@ -228,6 +231,10 @@ public class IndexerCoordinatorResource
     if (taskMasterLifecycle.getResourceManagementScheduler() == null) {
       return Response.noContent().build();
     }
-    return Response.ok(taskMasterLifecycle.getResourceManagementScheduler().getStats()).build();
+    ScalingStats foo = new ScalingStats(10);
+    foo.addProvisionEvent(new AutoScalingData<String>(Lists.newArrayList("boo", "doo"), Lists.newArrayList("a", "b")));
+    foo.addTerminateEvent(new AutoScalingData<String>(Lists.newArrayList("d", "e"), Lists.newArrayList("f", "g")));
+    return Response.ok(foo).build();
+    //return Response.ok(taskMasterLifecycle.getResourceManagementScheduler().getStats()).build();
   }
 }

merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/ScalingStats.java

 package com.metamx.druid.merger.coordinator.scaling;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.collect.Lists;
 import com.google.common.collect.MinMaxPriorityQueue;
 import org.joda.time.DateTime;
@@ -20,7 +21,6 @@ public class ScalingStats
   }
 
   private static final Comparator<ScalingEvent> comparator = new Comparator<ScalingEvent>()
-
   {
     @Override
     public int compare(ScalingEvent s1, ScalingEvent s2)
@@ -29,11 +29,13 @@ public class ScalingStats
     }
   };
 
-  private final MinMaxPriorityQueue<ScalingEvent> recentNodes;
+  private final Object lock = new Object();
+
+  private final MinMaxPriorityQueue<ScalingEvent> recentEvents;
 
   public ScalingStats(int capacity)
   {
-    this.recentNodes = MinMaxPriorityQueue
+    this.recentEvents = MinMaxPriorityQueue
         .orderedBy(comparator)
         .maximumSize(capacity)
         .create();
@@ -41,32 +43,38 @@ public class ScalingStats
 
   public void addProvisionEvent(AutoScalingData data)
   {
-    recentNodes.add(
-        new ScalingEvent(
-            data,
-            new DateTime(),
-            EVENT.PROVISION
-        )
-    );
+    synchronized (lock) {
+      recentEvents.add(
+          new ScalingEvent(
+              data,
+              new DateTime(),
+              EVENT.PROVISION
+          )
+      );
+    }
   }
 
   public void addTerminateEvent(AutoScalingData data)
   {
-    recentNodes.add(
-        new ScalingEvent(
-            data,
-            new DateTime(),
-            EVENT.TERMINATE
-        )
-    );
+    synchronized (lock) {
+      recentEvents.add(
+          new ScalingEvent(
+              data,
+              new DateTime(),
+              EVENT.TERMINATE
+          )
+      );
+    }
   }
 
-  @JsonProperty
+  @JsonValue
   public List<ScalingEvent> toList()
   {
-    List<ScalingEvent> retVal = Lists.newArrayList(recentNodes);
-    Collections.sort(retVal, comparator);
-    return retVal;
+    synchronized (lock) {
+      List<ScalingEvent> retVal = Lists.newArrayList(recentEvents);
+      Collections.sort(retVal, comparator);
+      return retVal;
+    }
   }
 
   public static class ScalingEvent

merger/src/main/resources/static/index.html → merger/src/main/resources/indexer_static/console.html

@@ -26,26 +26,34 @@
     <meta name="Description" content="Druid Indexer Coordinator Console"/>
 
     <style type="text/css">@import "css/style.css";</style>
+    <style type="text/css">@import "css/demo_table.css";</style>
 
     <script type="text/javascript" src="js/underscore-1.2.2.js"></script>
     <script type="text/javascript" src="js/jquery-1.8.3.js"></script>
-    <script type="text/javascript" src="js/indexer-0.0.1.js"></script>
+    <script type="text/javascript" src="js/jquery.dataTables-1.8.2.js"></script>
+    <script type="text/javascript" src="js/druidTable-0.0.1.js"></script>
+    <script type="text/javascript" src="js/tablehelper-0.0.2.js"></script>
+    <script type="text/javascript" src="js/console-0.0.1.js"></script>
 </head>
 
 <body>
 <div class="container">
     <div class="heading">Coordinator Console</div>
     <h2>Running Tasks</h2>
-    <div id="runningTasks"></div>
+    <div class="running_loading">Loading Running Tasks... this may take a few minutes</div>
+    <table id="runningTable"></table>
 
     <h2>Pending Tasks</h2>
-    <div id="pendingTasks"></div>
+    <div class="pending_loading">Loading Pending Tasks... this may take a few minutes</div>
+    <table id="pendingTable"></table>
 
     <h2>Workers</h2>
-    <div id="workers"></div>
+    <div class="workers_loading">Loading Workers... this may take a few minutes</div>
+    <table id="workerTable"></table>
 
     <h2>Event Log</h2>
-    <div id="events"></div>
+    <div class="events_loading">Loading Event Log... this may take a few minutes</div>
+    <table id="eventTable"></table>
 </div>
 </body>
 </html>
\ No newline at end of file

merger/src/main/resources/indexer_static/js/console-0.0.1.js

+// requires tableHelper
+
+var oTable = [];
+
+$(document).ready(function() {
+  $.get('/mmx/merger/v1/runningTasks', function(data) {
+    $('.running_loading').hide();
+    buildTable(data, $('#runningTable'), ["segments"]);
+  });
+
+  $.get('/mmx/merger/v1/pendingTasks', function(data) {
+    $('.pending_loading').hide();
+    buildTable(data, $('#pendingTable'), ["segments"]);
+  });
+
+  $.get('/mmx/merger/v1/workers', function(data) {
+    $('.workers_loading').hide();
+    buildTable(data, $('#workerTable'));
+  });
+
+  $.get('/mmx/merger/v1/scaling', function(data) {
+    $('.events_loading').hide();
+    buildTable(data, $('#eventTable'));
+  });
+});
\ No newline at end of file

merger/src/main/resources/static/css/style.css

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * HTML
- */
-
-body {
-	font: 14px Helvetica, Arial, Verdana, sans-serif;
-	color: #333;
-	background-color: #fff;
-}
-
-.container {
-  margin: 10px;
-  padding: 0 0 20px 0;
-}
-
-ul {
-  list-style: none;
-  margin: 0;
-  padding: 0;
-}
-
-a {
-  cursor: pointer;
-  text-decoration: underline;
-  margin: 0 5px 0 5px;
-  color:blue;
-}
-
-a:visited {
-  color: purple;
-}
-
-a:active {
-  color: red;
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * BODY
- */
-
-.heading {
-  background-color: #232B4A;
-  color: #FF6000;
-  padding: 10px;
-  margin: 10px 0;
-}
\ No newline at end of file

merger/src/main/resources/static/js/indexer-0.0.1.js

-$(document).ready(function() {
-  $.getJSON("/mmx/merger/v1/runningTasks", function(data) {
-    $('#pendingTasks').append('<p>' + JSON.stringify(data) + '</p>');
-  });
-
-  $.getJSON("/mmx/merger/v1/pendingTasks", function(data) {
-    $('#runningTasks').append('<p>' + JSON.stringify(data) + '</p>');
-  });
-
-  $.getJSON("/mmx/merger/v1/workers", function(data) {
-    $('#workers').append('<p>' + JSON.stringify(data) + '</p>');
-  });
-
-  $.getJSON("/mmx/merger/v1/scaling", function(data) {
-    $('#events').append('<p>' + JSON.stringify(data) + '</p>');
-  });
-});
\ No newline at end of file

server/src/main/resources/static/js/tablehelper-0.0.1.js → server/src/main/resources/static/js/tablehelper-0.0.2.js

-// requires dataTables
-
-var oTable = null;
+// requires jQuery, druidTable, dataTables
+var oTable = [];
 
 // flattens JSON from Druid and builds a table row per segment
-function initTables(data) {
-  var resultTable = new DruidTable();
-  var row = 0;
+function buildTable(data, el, dontDisplay, table, row) {
+  table = typeof table !== 'undefined' ? table : new DruidTable();
+  row = typeof row !== 'undefined' ? row : 0;
+  dontDisplay = typeof dontDisplay !== 'undefined' ? dontDisplay : [];
+
+  if (!Array.isArray(data) || data.length == 0) {
+    return;
+  }
+
+  if (oTable[el.attr('id')] != null) {
+    oTable[el.attr('id')].fnDestroy();
+    el.empty();
+  }
 
   // parse JSON
-  for (var entry in data) {
-    // build server table
-    for (var field in data[entry]) {
-      if (!(data[entry][field] instanceof Object)) {
-        resultTable.setCell(row, field, data[entry][field]);
-      }
-    }
+  for (var item in data) {
+    setTable(data[item], el, dontDisplay, table, row);
     row++;
   }
 
-  resultTable.toHTMLTable($('#result_table'));
+  table.toHTMLTable(el);
+  initDataTable(el);
+}
+
+function setTable(data, el, dontDisplay, table, row) {
+  for (var field in data) {
+    if (_.contains(dontDisplay, field)) {
+      // do nothing
+    } else if (Array.isArray(data[field])) {
+      table.setCell(row, field, JSON.stringify(data[field]));
+    } else if (!(data[field] instanceof Object)) {
+      table.setCell(row, field, data[field]);
+    } else {
+      setTable(data[field], el, dontDisplay, table, row);
+    }
+  }
 }
 
 function initDataTable(el) {
   // dataTable stuff (http://www.datatables.net/)
   var asInitVals = [];
 
-  oTable = el.dataTable({
+  oTable[el.attr('id')] = el.dataTable({
     "oLanguage": {
       "sSearch": "Search all columns:"
     },
@@ -34,13 +53,12 @@ function initDataTable(el) {
       "bRegex": true
     },
     "sPaginationType": "full_numbers",
-    "bProcessing": true,
-    "bDeferRender": true
+    "bProcessing": true
   });
 
-
   $("thead input").keyup(function() {
-      oTable.fnFilter(this.value, oTable.children("thead").find("input").index(this), true);
+      var tbl = oTable[$(this).parents('table').attr('id')];
+      tbl.fnFilter(this.value, tbl.children("thead").find("input").index(this), true);
   });
 
   $("thead input").each(function(i) {
@@ -60,14 +78,4 @@ function initDataTable(el) {
       this.value = asInitVals[$("thead input").index(this)];
     }
   });
-}
-
-function buildTable(data, el) {
-  if (oTable != null) {
-    oTable.fnDestroy();
-    el.empty();
-  }
-
-  initTables(data);
-  initDataTable(el);
 }
\ No newline at end of file

server/src/main/resources/static/view.html

@@ -33,7 +33,7 @@
   <script type="text/javascript" src="js/jquery-1.8.3.js"></script>
   <script type="text/javascript" src="js/jquery.dataTables-1.8.2.js"></script>
   <script type="text/javascript" src="js/druidTable-0.0.1.js"></script>
-  <script type="text/javascript" src="js/tablehelper-0.0.1.js"></script>
+  <script type="text/javascript" src="js/tablehelper-0.0.2.js"></script>
   <script type="text/javascript" src="js/handlers-0.0.1.js"></script>
 </head>