1) Adjustments to allow for the addition of new Runnables via modules. ...

1) Adjustments to allow for the addition of new Runnables via modules.  Modules should implement CliCommandCreator and add their stuff there.

common/src/main/java/io/druid/guice/DruidSecondaryModule.java

@@ -68,6 +68,7 @@ public class DruidSecondaryModule implements Module
   @Override
   public void configure(Binder binder)
   {
+    binder.requireExplicitBindings();
     binder.install(new DruidGuiceExtensions());
     binder.bind(Properties.class).toInstance(properties);
     binder.bind(ConfigurationObjectFactory.class).toInstance(factory);

server/src/main/java/io/druid/server/initialization/Initialization.java

@@ -22,117 +22,126 @@ package io.druid.server.initialization;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.common.io.ByteStreams;
-import com.google.inject.Binder;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.Module;
 import com.metamx.common.ISE;
 import com.metamx.common.logger.Logger;
-import io.druid.guice.DruidGuiceExtensions;
 import io.druid.guice.DruidSecondaryModule;
-import io.druid.guice.JsonConfigProvider;
 import io.druid.guice.annotations.Json;
 import io.druid.guice.annotations.Smile;
 import io.druid.initialization.DruidModule;
-import io.druid.jackson.JacksonModule;
 import io.tesla.aether.TeslaAether;
 import io.tesla.aether.internal.DefaultTeslaAether;
 import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.collection.CollectRequest;
+import org.eclipse.aether.graph.Dependency;
+import org.eclipse.aether.graph.DependencyFilter;
+import org.eclipse.aether.graph.DependencyNode;
+import org.eclipse.aether.resolution.DependencyRequest;
+import org.eclipse.aether.util.artifact.JavaScopes;
+import org.eclipse.aether.util.filter.DependencyFilterUtils;
 
 import java.io.PrintStream;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.ServiceLoader;
+import java.util.Set;
 
 /**
  */
 public class Initialization
 {
   private static final Logger log = new Logger(Initialization.class);
+  private static final Map<String, ClassLoader> loadersMap = Maps.newHashMap();
 
-  private static final List<String> exclusions = Arrays.asList(
+  private static final Set<String> exclusions = Sets.newHashSet(
       "io.druid",
       "com.metamx.druid"
   );
 
-
-  public static Injector makeInjector(final Object... modules)
-  {
-    final Injector baseInjector = Guice.createInjector(
-        new DruidGuiceExtensions(),
-        new JacksonModule(),
-        new PropertiesModule("runtime.properties"),
-        new ConfigModule(),
-        new Module()
-        {
-          @Override
-          public void configure(Binder binder)
-          {
-            binder.bind(DruidSecondaryModule.class);
-            JsonConfigProvider.bind(binder, "druid.extensions", ExtensionsConfig.class);
-
-            for (Object module : modules) {
-              if (module instanceof Class) {
-                binder.bind((Class) module);
-              }
-            }
-          }
-        }
-    );
-
-    ModuleList actualModules = new ModuleList(baseInjector);
-    actualModules.addModule(DruidSecondaryModule.class);
-    for (Object module : modules) {
-      actualModules.addModule(module);
-    }
-
-    addExtensionModules(baseInjector.getInstance(ExtensionsConfig.class), actualModules);
-
-    return Guice.createInjector(actualModules.getModules());
-  }
-
-  private static void addExtensionModules(ExtensionsConfig config, ModuleList actualModules)
+  public synchronized static <T> List<T> getFromExtensions(ExtensionsConfig config, Class<T> clazz)
   {
     final TeslaAether aether = getAetherClient(config);
+    List<T> retVal = Lists.newArrayList();
 
     for (String coordinate : config.getCoordinates()) {
       log.info("Loading extension[%s]", coordinate);
       try {
-        final List<Artifact> artifacts = aether.resolveArtifacts(coordinate);
-        List<URL> urls = Lists.newArrayListWithExpectedSize(artifacts.size());
-        for (Artifact artifact : artifacts) {
-          if (!exclusions.contains(artifact.getGroupId())) {
-            urls.add(artifact.getFile().toURI().toURL());
+        ClassLoader loader = loadersMap.get(coordinate);
+        if (loader == null) {
+          final CollectRequest collectRequest = new CollectRequest();
+          collectRequest.setRoot(new Dependency(new DefaultArtifact(coordinate), JavaScopes.RUNTIME));
+          DependencyRequest dependencyRequest = new DependencyRequest(
+              collectRequest,
+              DependencyFilterUtils.andFilter(
+                  DependencyFilterUtils.classpathFilter(JavaScopes.RUNTIME),
+                  new DependencyFilter()
+                  {
+                    @Override
+                    public boolean accept(DependencyNode node, List<DependencyNode> parents)
+                    {
+                      if (accept(node.getArtifact())) {
+                        return false;
+                      }
+
+                      for (DependencyNode parent : parents) {
+                        if (accept(parent.getArtifact())) {
+                          return false;
+                        }
+                      }
+
+                      return true;
+                    }
+
+                    private boolean accept(final Artifact artifact)
+                    {
+                      return exclusions.contains(artifact.getGroupId());
+                    }
+                  }
+              )
+          );
+
+          final List<Artifact> artifacts = aether.resolveArtifacts(dependencyRequest);
+          List<URL> urls = Lists.newArrayListWithExpectedSize(artifacts.size());
+          for (Artifact artifact : artifacts) {
+            if (!exclusions.contains(artifact.getGroupId())) {
+              urls.add(artifact.getFile().toURI().toURL());
+            }
+            else {
+              log.error("Skipped Artifact[%s]", artifact);
+            }
           }
-          else {
-            log.debug("Skipped Artifact[%s]", artifact);
+
+          for (URL url : urls) {
+            log.error("Added URL[%s]", url);
           }
-        }
 
-        for (URL url : urls) {
-          log.debug("Added URL[%s]", url);
+          loader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader());
+          loadersMap.put(coordinate, loader);
         }
 
-        ClassLoader loader = new URLClassLoader(
-            urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader()
-        );
-
-        final ServiceLoader<DruidModule> serviceLoader = ServiceLoader.load(DruidModule.class, loader);
+        final ServiceLoader<T> serviceLoader = ServiceLoader.load(clazz, loader);
 
-        for (DruidModule module : serviceLoader) {
+        for (T module : serviceLoader) {
           log.info("Adding extension module[%s]", module.getClass());
-          actualModules.addModule(module);
+          retVal.add(module);
         }
       }
       catch (Exception e) {
         throw Throwables.propagate(e);
       }
     }
+
+    return retVal;
   }
 
   private static DefaultTeslaAether getAetherClient(ExtensionsConfig config)
@@ -162,6 +171,22 @@ public class Initialization
     }
   }
 
+  public static Injector makeInjectorWithModules(final Injector baseInjector, List<Object> modules)
+  {
+    ModuleList actualModules = new ModuleList(baseInjector);
+    actualModules.addModule(DruidSecondaryModule.class);
+    for (Object module : modules) {
+      actualModules.addModule(module);
+    }
+
+    final ExtensionsConfig config = baseInjector.getInstance(ExtensionsConfig.class);
+    for (DruidModule module : Initialization.getFromExtensions(config, DruidModule.class)) {
+      actualModules.addModule(module);
+    }
+
+    return Guice.createInjector(actualModules.getModules());
+  }
+
   private static class ModuleList
   {
     private final Injector baseInjector;
@@ -217,5 +242,4 @@ public class Initialization
       return module;
     }
   }
-
-  }
+}

services/src/main/java/io/druid/cli/CliBroker.java

@@ -19,7 +19,7 @@
 
 package io.druid.cli;
 
-import com.google.inject.Injector;
+import com.google.common.collect.ImmutableList;
 import com.metamx.common.logger.Logger;
 import io.airlift.command.Command;
 import io.druid.client.cache.CacheMonitor;
@@ -35,10 +35,11 @@ import io.druid.guice.annotations.Client;
 import io.druid.server.ClientQuerySegmentWalker;
 import io.druid.server.StatusResource;
 import io.druid.server.initialization.EmitterModule;
-import io.druid.server.initialization.Initialization;
 import io.druid.server.initialization.JettyServerModule;
 import io.druid.server.metrics.MetricsModule;
 
+import java.util.List;
+
 /**
  */
 @Command(
@@ -55,22 +56,22 @@ public class CliBroker extends ServerRunnable
   }
 
   @Override
-  protected Injector getInjector()
+  protected List<Object> getModules()
   {
-    return Initialization.makeInjector(
-            new LifecycleModule(),
-            EmitterModule.class,
-            HttpClientModule.global(),
-            CuratorModule.class,
-            new MetricsModule().register(CacheMonitor.class),
-            new ServerModule(),
-            new JettyServerModule(new QueryJettyServerInitializer())
-                .addResource(StatusResource.class),
-            new QueryableModule(ClientQuerySegmentWalker.class),
-            new QueryToolChestModule(),
-            new ServerViewModule(),
-            new HttpClientModule("druid.broker.http", Client.class),
-            new BrokerModule()
+    return ImmutableList.<Object>of(
+        new LifecycleModule(),
+        EmitterModule.class,
+        HttpClientModule.global(),
+        CuratorModule.class,
+        new MetricsModule().register(CacheMonitor.class),
+        new ServerModule(),
+        new JettyServerModule(new QueryJettyServerInitializer())
+            .addResource(StatusResource.class),
+        new QueryableModule(ClientQuerySegmentWalker.class),
+        new QueryToolChestModule(),
+        new ServerViewModule(),
+        new HttpClientModule("druid.broker.http", Client.class),
+        new BrokerModule()
     );
   }
 }

services/src/main/java/io/druid/cli/CliCoordinator.java

@@ -19,6 +19,7 @@
 
 package io.druid.cli;
 
+import com.google.common.collect.ImmutableList;
 import com.google.inject.Injector;
 import com.google.inject.servlet.GuiceFilter;
 import com.metamx.common.logger.Logger;
@@ -39,7 +40,6 @@ import io.druid.server.http.InfoResource;
 import io.druid.server.http.MasterResource;
 import io.druid.server.http.RedirectFilter;
 import io.druid.server.initialization.EmitterModule;
-import io.druid.server.initialization.Initialization;
 import io.druid.server.initialization.JettyServerInitializer;
 import io.druid.server.initialization.JettyServerModule;
 import io.druid.server.master.DruidMaster;
@@ -55,6 +55,8 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.servlets.GzipFilter;
 
+import java.util.List;
+
 /**
  */
 @Command(
@@ -71,9 +73,9 @@ public class CliCoordinator extends ServerRunnable
   }
 
   @Override
-  protected Injector getInjector()
+  protected List<Object> getModules()
   {
-    return Initialization.makeInjector(
+    return ImmutableList.<Object>of(
         new LifecycleModule().register(DruidMaster.class),
         EmitterModule.class,
         HttpClientModule.global(),

services/src/main/java/io/druid/cli/CliHistorical.java

@@ -19,7 +19,7 @@
 
 package io.druid.cli;
 
-import com.google.inject.Injector;
+import com.google.common.collect.ImmutableList;
 import com.metamx.common.logger.Logger;
 import io.airlift.command.Command;
 import io.druid.curator.CuratorModule;
@@ -38,11 +38,12 @@ import io.druid.server.StatusResource;
 import io.druid.server.coordination.ServerManager;
 import io.druid.server.coordination.ZkCoordinator;
 import io.druid.server.initialization.EmitterModule;
-import io.druid.server.initialization.Initialization;
 import io.druid.server.initialization.JettyServerModule;
 import io.druid.server.metrics.MetricsModule;
 import io.druid.server.metrics.ServerMonitor;
 
+import java.util.List;
+
 /**
  */
 @Command(
@@ -59,9 +60,9 @@ public class CliHistorical extends ServerRunnable
   }
 
   @Override
-  protected Injector getInjector()
+  protected List<Object> getModules()
   {
-    return Initialization.makeInjector(
+    return ImmutableList.<Object>of(
         new LifecycleModule().register(ZkCoordinator.class),
         EmitterModule.class,
         HttpClientModule.global(),

services/src/main/java/io/druid/cli/CliMiddleManager.java

@@ -19,6 +19,7 @@
 
 package io.druid.cli;
 
+import com.google.common.collect.ImmutableList;
 import com.google.inject.Injector;
 import com.google.inject.servlet.GuiceFilter;
 import com.metamx.common.logger.Logger;
@@ -34,7 +35,6 @@ import io.druid.indexing.worker.WorkerTaskMonitor;
 import io.druid.indexing.worker.http.WorkerResource;
 import io.druid.server.StatusResource;
 import io.druid.server.initialization.EmitterModule;
-import io.druid.server.initialization.Initialization;
 import io.druid.server.initialization.JettyServerInitializer;
 import io.druid.server.initialization.JettyServerModule;
 import io.druid.server.metrics.MetricsModule;
@@ -47,6 +47,8 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.servlets.GzipFilter;
 
+import java.util.List;
+
 /**
  */
 @Command(
@@ -63,9 +65,9 @@ public class CliMiddleManager extends ServerRunnable
   }
 
   @Override
-  protected Injector getInjector()
+  protected List<Object> getModules()
   {
-    return Initialization.makeInjector(
+    return ImmutableList.<Object>of(
         new LifecycleModule().register(WorkerTaskMonitor.class),
         EmitterModule.class,
         HttpClientModule.global(),

services/src/main/java/io/druid/cli/CliOverlord.java

@@ -19,6 +19,7 @@
 
 package io.druid.cli;
 
+import com.google.common.collect.ImmutableList;
 import com.google.inject.Injector;
 import com.google.inject.servlet.GuiceFilter;
 import com.metamx.common.logger.Logger;
@@ -38,7 +39,6 @@ import io.druid.indexing.coordinator.http.IndexerCoordinatorResource;
 import io.druid.server.StatusResource;
 import io.druid.server.http.RedirectFilter;
 import io.druid.server.initialization.EmitterModule;
-import io.druid.server.initialization.Initialization;
 import io.druid.server.initialization.JettyServerInitializer;
 import io.druid.server.initialization.JettyServerModule;
 import io.druid.server.metrics.MetricsModule;
@@ -54,6 +54,8 @@ import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.servlets.GzipFilter;
 import org.eclipse.jetty.util.resource.ResourceCollection;
 
+import java.util.List;
+
 /**
  */
 @Command(
@@ -70,9 +72,9 @@ public class CliOverlord extends ServerRunnable
   }
 
   @Override
-  protected Injector getInjector()
+  protected List<Object> getModules()
   {
-    return Initialization.makeInjector(
+    return ImmutableList.<Object>of(
         new LifecycleModule(),
         EmitterModule.class,
         HttpClientModule.global(),

services/src/main/java/io/druid/cli/CliPeon.java

@@ -20,6 +20,8 @@
 package io.druid.cli;
 
 import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.metamx.common.lifecycle.Lifecycle;
 import com.metamx.common.logger.Logger;
@@ -70,34 +72,45 @@ public class CliPeon implements Runnable
   @Option(name = "--nodeType", title = "nodeType", description = "Set the node type to expose on ZK")
   public String nodeType = "indexer-executor";
 
+  private Injector injector;
+
+  @Inject
+  public void configure(Injector injector)
+  {
+    this.injector = injector;
+  }
+
   private static final Logger log = new Logger(CliPeon.class);
 
   protected Injector getInjector()
   {
-    return Initialization.makeInjector(
-        new LifecycleModule(),
-        EmitterModule.class,
-        HttpClientModule.global(),
-        CuratorModule.class,
-        new MetricsModule(),
-        new ServerModule(),
-        new JettyServerModule(new QueryJettyServerInitializer())
-            .addResource(StatusResource.class)
-            .addResource(ChatHandlerResource.class),
-        new DiscoveryModule(),
-        new ServerViewModule(),
-        new StorageNodeModule(nodeType),
-        new DataSegmentPusherModule(),
-        new AnnouncerModule(),
-        new DruidProcessingModule(),
-        new QueryableModule(ThreadPoolTaskRunner.class),
-        new QueryRunnerFactoryModule(),
-        new IndexingServiceDiscoveryModule(),
-        new AWSModule(),
-        new PeonModule(
-            new ExecutorLifecycleConfig()
-                .setTaskFile(new File(taskAndStatusFile.get(0)))
-                .setStatusFile(new File(taskAndStatusFile.get(1)))
+    return Initialization.makeInjectorWithModules(
+        injector,
+        ImmutableList.of(
+            new LifecycleModule(),
+            EmitterModule.class,
+            HttpClientModule.global(),
+            CuratorModule.class,
+            new MetricsModule(),
+            new ServerModule(),
+            new JettyServerModule(new QueryJettyServerInitializer())
+                .addResource(StatusResource.class)
+                .addResource(ChatHandlerResource.class),
+            new DiscoveryModule(),
+            new ServerViewModule(),
+            new StorageNodeModule(nodeType),
+            new DataSegmentPusherModule(),
+            new AnnouncerModule(),
+            new DruidProcessingModule(),
+            new QueryableModule(ThreadPoolTaskRunner.class),
+            new QueryRunnerFactoryModule(),
+            new IndexingServiceDiscoveryModule(),
+            new AWSModule(),
+            new PeonModule(
+                new ExecutorLifecycleConfig()
+                    .setTaskFile(new File(taskAndStatusFile.get(0)))
+                    .setStatusFile(new File(taskAndStatusFile.get(1)))
+            )
         )
     );
   }

services/src/main/java/io/druid/cli/CliRealtime.java

@@ -19,7 +19,7 @@
 
 package io.druid.cli;
 
-import com.google.inject.Injector;
+import com.google.common.collect.ImmutableList;
 import com.metamx.common.logger.Logger;
 import io.airlift.command.Command;
 import io.druid.curator.CuratorModule;
@@ -39,10 +39,11 @@ import io.druid.guice.StorageNodeModule;
 import io.druid.segment.realtime.RealtimeManager;
 import io.druid.server.StatusResource;
 import io.druid.server.initialization.EmitterModule;
-import io.druid.server.initialization.Initialization;
 import io.druid.server.initialization.JettyServerModule;
 import io.druid.server.metrics.MetricsModule;
 
+import java.util.List;
+
 /**
  */
 @Command(
@@ -59,9 +60,9 @@ public class CliRealtime extends ServerRunnable
   }
 
   @Override
-  protected Injector getInjector()
+  protected List<Object> getModules()
   {
-    return Initialization.makeInjector(
+    return ImmutableList.<Object>of(
         new LifecycleModule(),
         EmitterModule.class,
         DbConnectorModule.class,

services/src/main/java/io/druid/cli/CliRealtimeExample.java

@@ -19,7 +19,7 @@
 
 package io.druid.cli;
 
-import com.google.inject.Injector;
+import com.google.common.collect.ImmutableList;
 import com.metamx.common.logger.Logger;
 import druid.examples.guice.RealtimeExampleModule;
 import io.airlift.command.Command;
@@ -32,9 +32,10 @@ import io.druid.guice.StorageNodeModule;
 import io.druid.segment.realtime.RealtimeManager;
 import io.druid.server.StatusResource;
 import io.druid.server.initialization.EmitterModule;
-import io.druid.server.initialization.Initialization;
 import io.druid.server.initialization.JettyServerModule;
 
+import java.util.List;
+
 /**
  */
 @Command(
@@ -51,9 +52,9 @@ public class CliRealtimeExample extends ServerRunnable
   }
 
   @Override
-  protected Injector getInjector()
+  protected List<Object> getModules()
   {
-    return Initialization.makeInjector(
+    return ImmutableList.of(
         new LifecycleModule(),
         EmitterModule.class,
         DruidProcessingModule.class,

services/src/main/java/io/druid/cli/Main.java

 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -19,9 +19,23 @@
 
 package io.druid.cli;
 
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
 import io.airlift.command.Cli;
 import io.airlift.command.Help;
 import io.airlift.command.ParseException;
+import io.druid.guice.DruidGuiceExtensions;
+import io.druid.guice.DruidSecondaryModule;
+import io.druid.guice.JsonConfigProvider;
+import io.druid.jackson.JacksonModule;
+import io.druid.server.initialization.ConfigModule;
+import io.druid.server.initialization.ExtensionsConfig;
+import io.druid.server.initialization.Initialization;
+import io.druid.server.initialization.PropertiesModule;
+
+import java.util.List;
 
 /**
  */
@@ -54,9 +68,19 @@ public class Main
            .withDefaultCommand(Help.class)
            .withCommands(CliPeon.class);
 
+    final Injector injector = makeStartupInjector();
+    final ExtensionsConfig config = injector.getInstance(ExtensionsConfig.class);
+    final List<CliCommandCreator> extensionCommands = Initialization.getFromExtensions(config, CliCommandCreator.class);
+
+    for (CliCommandCreator creator : extensionCommands) {
+      creator.addCommands(builder);
+    }
+
     final Cli<Runnable> cli = builder.build();
     try {
-      cli.parse(args).run();
+      final Runnable command = cli.parse(args);
+      injector.injectMembers(command);
+      command.run();
     }
     catch (ParseException e) {
       System.out.println("ERROR!!!!");
@@ -65,4 +89,23 @@ public class Main
       cli.parse(new String[]{"help"}).run();
     }
   }
+
+  public static Injector makeStartupInjector()
+  {
+    return Guice.createInjector(
+        new DruidGuiceExtensions(),
+        new JacksonModule(),
+        new PropertiesModule("runtime.properties"),
+        new ConfigModule(),
+        new Module()
+        {
+          @Override
+          public void configure(Binder binder)
+          {
+            binder.bind(DruidSecondaryModule.class);
+            JsonConfigProvider.bind(binder, "druid.extensions", ExtensionsConfig.class);
+          }
+        }
+    );
+  }
 }

services/src/main/java/io/druid/cli/ServerRunnable.java

@@ -20,10 +20,14 @@
 package io.druid.cli;
 
 import com.google.common.base.Throwables;
+import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.metamx.common.lifecycle.Lifecycle;
 import com.metamx.common.logger.Logger;
 import io.druid.initialization.LogLevelAdjuster;
+import io.druid.server.initialization.Initialization;
+
+import java.util.List;
 
 /**
  */
@@ -31,12 +35,20 @@ public abstract class ServerRunnable implements Runnable
 {
   private final Logger log;
 
+  private Injector baseInjector;
+
   public ServerRunnable(Logger log)
   {
     this.log = log;
   }
 
-  protected abstract Injector getInjector();
+  @Inject
+  public void configure(Injector injector)
+  {
+    this.baseInjector = injector;
+  }
+
+  protected abstract List<Object> getModules();
 
   @Override
   public void run()
@@ -44,7 +56,8 @@ public abstract class ServerRunnable implements Runnable
     try {
       LogLevelAdjuster.register();
 
-      final Lifecycle lifecycle = getInjector().getInstance(Lifecycle.class);
+      final Injector injector = Initialization.makeInjectorWithModules(baseInjector, getModules());
+      final Lifecycle lifecycle = injector.getInstance(Lifecycle.class);
 
       try {
         lifecycle.start();