tidied up build, added bash shell script to manage HTTP server

core/pom.xml

@@ -90,11 +90,29 @@
                 </configuration>
                 <executions>
                     <execution>
-                        <id>make-assembly</id>
+                        <id>create-jar</id>
                         <phase>package</phase>
                         <goals>
                             <goal>single</goal>
                         </goals>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>src/main/assembly/assembly.xml</descriptor>
+                            </descriptors>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>create-binary</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>src/main/assembly/binaries.xml</descriptor>
+                            </descriptors>
+                            <finalName>questdb-${project.version}</finalName>
+                        </configuration>
                     </execution>
                 </executions>
             </plugin>

core/src/main/assembly/assembly.xml

@@ -25,14 +25,12 @@
     <id>all</id>
     <formats>
         <format>tar.gz</format>
-        <format>tar.bz2</format>
-        <format>zip</format>
     </formats>
     <includeBaseDirectory>false</includeBaseDirectory>
     <fileSets>
         <fileSet>
             <directory>${project.basedir}/src</directory>
-            <outputDirectory>/src</outputDirectory>
+            <outputDirectory>src</outputDirectory>
         </fileSet>
         <fileSet>
             <directory>${project.build.directory}/site</directory>

core/src/main/assembly/binaries.xml

+<!--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  ~    ___                  _   ____  ____
+  ~   / _ \ _   _  ___  ___| |_|  _ \| __ )
+  ~  | | | | | | |/ _ \/ __| __| | | |  _ \
+  ~  | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+  ~   \__\_\\__,_|\___||___/\__|____/|____/
+  ~
+  ~ Copyright (C) 2014-2016 Appsicle
+  ~
+  ~ This program is free software: you can redistribute it and/or  modify
+  ~ it under the terms of the GNU Affero General Public License, version 3,
+  ~ as published by the Free Software Foundation.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU Affero General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Affero General Public License
+  ~ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+  ~
+  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-->
+
+<assembly>
+    <id>bin</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>questdb-${artifact.version}</outputDirectory>
+            <outputFileNameMapping>questdb.jar</outputFileNameMapping>
+            <useProjectArtifact>true</useProjectArtifact>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+    <fileSets>
+        <fileSet>
+            <directory>src/main/bin</directory>
+            <outputDirectory>questdb-${artifact.version}</outputDirectory>
+            <fileMode>755</fileMode>
+        </fileSet>
+    </fileSets>
+</assembly>
\ No newline at end of file

core/src/main/bin/questdb.sh

+#!/bin/sh
+
+export QDB_PROCESS_LABEL="QuestDB-Runtime-66535"
+export QDB_MAX_STOP_ATTEMPTS=5;
+
+function start {
+    # check that JAVA_HOME is defined
+    if [ "$JAVA_HOME" = "" ]; then
+        echo "JAVA_HOME is undefined"
+        exit 55
+    fi
+
+    # check that Java binary is executable
+    JAVA=${JAVA_HOME}/bin/java
+    if [ ! -x "$JAVA" ]; then
+        echo "$JAVA is not executable"
+        exit 55;
+    fi
+
+    # check the state of root directory
+    if [ "$1" = "" ]; then
+        QDB_ROOT="qdbroot"
+    else
+        QDB_ROOT="$1"
+    fi
+
+    # create root directory if it does not exist
+    if [ ! -d "$QDB_ROOT" ]; then
+        echo "Created QuestDB ROOT directory: $QDB_ROOT"
+        mkdir -p ${QDB_ROOT}
+    fi
+
+    QDB_LOG=${QDB_ROOT}/log
+    mkdir -p ${QDB_LOG}
+
+    JAVA_LIB="`dirname $0`/questdb.jar"
+
+    JAVA_OPTS="
+    -D$QDB_PROCESS_LABEL
+    -da -Dnoebug
+    -XX:+PrintGCApplicationStoppedTime
+    -XX:+PrintSafepointStatistics
+    -XX:PrintSafepointStatisticsCount=1
+    -XX:+UseParNewGC
+    -XX:+UseConcMarkSweepGC
+    -XX:+PrintGCDetails
+    -XX:+PrintGCTimeStamps
+    -XX:+PrintGCDateStamps
+    -XX:+UnlockDiagnosticVMOptions
+    -XX:GuaranteedSafepointInterval=90000000
+    -XX:-UseBiasedLocking
+    -XX:BiasedLockingStartupDelay=0"
+
+    JAVA_MAIN="com.questdb.BootstrapMain"
+
+    ${JAVA} ${JAVA_OPTS} -cp ${JAVA_LIB} ${JAVA_MAIN} ${QDB_ROOT} -f > ${QDB_LOG}/stdout.txt &
+    sleep 0.5
+}
+
+function export_pid {
+    export QDB_PID=`ps -ef | grep ${QDB_PROCESS_LABEL} | grep -v grep | tr -s " " | cut -d " " -f 3`
+}
+
+function query {
+    export_pid
+
+    if [[ "${QDB_PID}" = "" ]]; then
+        echo "Not running"
+    else
+        echo "PID: ${QDB_PID}"
+    fi
+}
+
+function stop {
+    export_pid
+    if [[ "${QDB_PID}" = "" ]]; then
+        echo "Not running"
+        exit 55
+    fi
+
+    OUR_PID=${QDB_PID}
+
+    count=${QDB_MAX_STOP_ATTEMPTS}
+    while [ "${QDB_PID}" != "" ] && [ ${count} -gt 0 ]; do
+        kill ${QDB_PID}
+        sleep 0.5
+        export_pid
+        count=$((count-1))
+    done
+
+    if [[ "${QDB_PID}" != "" ]]; then
+        kill -9 ${QDB_PID}
+        echo "Something is wrong. Process does not stop. Killing.."
+        export_pid
+    fi
+
+    if [[ "${QDB_PID}" != "" ]]; then
+        echo "Cannot stop ${QDB_PID}. Check permissions."
+    else
+        echo "Stopped ${OUR_PID}"
+    fi
+}
+
+function banner {
+    echo ''
+    echo '  ___                  _   ____  ____'
+    echo ' / _ \ _   _  ___  ___| |_|  _ \| __ )'
+    echo '| | | | | | |/ _ \/ __| __| | | |  _ \'
+    echo '| |_| | |_| |  __/\__ \ |_| |_| | |_) |'
+    echo ' \__\_\\\__,_|\___||___/\__|____/|____/'
+    echo
+}
+
+function usage {
+    echo "$0 start|stop|query [-f]"
+    exit 55
+}
+
+banner
+
+if [[ $# -gt 0 ]]; then
+    command=$1
+    shift
+
+    case ${command} in
+       start)
+        start
+        ;;
+       query)
+        query
+        ;;
+       stop)
+        stop
+        ;;
+       *)
+        usage
+        ;;
+    esac
+fi

core/src/main/java/com/questdb/BootstrapMain.java

@@ -35,14 +35,19 @@ import com.questdb.net.http.MimeTypes;
 import com.questdb.net.http.ServerConfiguration;
 import com.questdb.net.http.SimpleUrlMatcher;
 import com.questdb.net.http.handlers.*;
+import sun.misc.Signal;
+import sun.misc.SignalHandler;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Date;
 import java.util.EnumSet;
+import java.util.HashMap;
 
 import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES;
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
@@ -50,14 +55,14 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 class BootstrapMain {
 
     public static void main(String[] args) throws Exception {
-        System.out.printf("QDB HTTP Server 3.0%nCopyright (C) Appsicle 2014-2016, all rights reserved.%n%n");
+        System.err.printf("QuestDB HTTP Server 3.0%nCopyright (C) Appsicle 2014-2016, all rights reserved.%n%n");
         if (args.length < 1) {
-            System.out.println("Root directory name expected");
+            System.err.println("Root directory name expected");
             return;
         }
 
         if (Os.type == Os._32Bit) {
-            System.out.println("QDB requires 64-bit JVM");
+            System.err.println("QDB requires 64-bit JVM");
             return;
         }
 
@@ -68,7 +73,7 @@ class BootstrapMain {
         File conf = new File(dir, "conf/questdb.conf");
 
         if (!conf.exists()) {
-            System.out.println("Configuration file does not exist: " + conf);
+            System.err.println("Configuration file does not exist: " + conf);
             return;
         }
 
@@ -84,18 +89,36 @@ class BootstrapMain {
         matcher.put("/chk", new ExistenceCheckHandler(factory));
         matcher.setDefaultHandler(new StaticContentHandler(configuration.getHttpPublic(), new MimeTypes(configuration.getMimeTypes())));
 
-        HttpServer server = new HttpServer(configuration, matcher);
-        server.start(LogFactory.INSTANCE.getJobs(), configuration.getHttpQueueDepth());
-
         StringBuilder welcome = Misc.getThreadLocalBuilder();
-        welcome.append("Listening on ").append(configuration.getHttpIP()).append(':').append(configuration.getHttpPort());
-        if (configuration.getSslConfig().isSecure()) {
-            welcome.append(" [HTTPS]");
+        HttpServer server = new HttpServer(configuration, matcher);
+        if (!server.start(LogFactory.INSTANCE.getJobs(), configuration.getHttpQueueDepth())) {
+            welcome.append("Could not bind socket ").append(configuration.getHttpIP()).append(':').append(configuration.getHttpPort());
+            welcome.append(". Already running?");
+            System.err.println(welcome);
+            System.out.println(new Date() + " QuestDB failed to start");
         } else {
-            welcome.append(" [HTTP plain]");
-        }
+            welcome.append("Listening on ").append(configuration.getHttpIP()).append(':').append(configuration.getHttpPort());
+            if (configuration.getSslConfig().isSecure()) {
+                welcome.append(" [HTTPS]");
+            } else {
+                welcome.append(" [HTTP plain]");
+            }
+
+            System.err.println(welcome);
+            System.out.println(new Date() + " QuestDB is running");
+
+            // suppress HUP signal
+            Signal.handle(new Signal("HUP"), new SignalHandler() {
+                public void handle(Signal signal) {
+                }
+            });
 
-        System.out.println(welcome);
+            Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+                public void run() {
+                    System.out.println(new Date() + " QuestDB is shutting down");
+                }
+            }));
+        }
     }
 
     private static void configureLoggers(final ServerConfiguration configuration) {
@@ -123,55 +146,80 @@ class BootstrapMain {
     }
 
     private static void extractSite(String dir, boolean force) throws URISyntaxException, IOException {
+        System.out.println("Preparing content...");
         URL url = HttpServer.class.getResource("/site/");
-        final Path source = Paths.get(url.toURI());
-        final Path target = Paths.get(dir);
-        final EnumSet<FileVisitOption> walkOptions = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
-        final CopyOption[] copyOptions = new CopyOption[]{COPY_ATTRIBUTES, REPLACE_EXISTING};
-
-        if (force) {
-            File pub = new File(dir, "public");
-            if (pub.exists()) {
-                com.questdb.misc.Files.delete(pub);
-            }
+        String[] components = url.toURI().toString().split("!");
+        FileSystem fs = null;
+        final Path source;
+        final int sourceLen;
+        if (components.length > 1) {
+            fs = FileSystems.newFileSystem(URI.create(components[0]), new HashMap<String, Object>());
+            source = fs.getPath(components[1]);
+            sourceLen = source.toAbsolutePath().toString().length();
+        } else {
+            source = Paths.get(url.toURI());
+            sourceLen = source.toAbsolutePath().toString().length() + 1;
         }
 
-        Files.walkFileTree(source, walkOptions, Integer.MAX_VALUE, new FileVisitor<Path>() {
+        try {
+            final Path target = Paths.get(dir);
+            final EnumSet<FileVisitOption> walkOptions = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
+            final CopyOption[] copyOptions = new CopyOption[]{COPY_ATTRIBUTES, REPLACE_EXISTING};
 
-            private boolean skip = true;
+            if (force) {
+                File pub = new File(dir, "public");
+                if (pub.exists()) {
+                    com.questdb.misc.Files.delete(pub);
+                }
+            }
 
-            @Override
-            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
-                if (skip) {
-                    skip = false;
-                } else {
-                    Path newDirectory = target.resolve(source.relativize(dir));
-                    try {
-                        Files.copy(dir, newDirectory, copyOptions);
-                    } catch (FileAlreadyExistsException ignore) {
-                    } catch (IOException x) {
-                        return FileVisitResult.SKIP_SUBTREE;
+            Files.walkFileTree(source, walkOptions, Integer.MAX_VALUE, new FileVisitor<Path>() {
+
+                private boolean skip = true;
+
+                @Override
+                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
+                    if (skip) {
+                        skip = false;
+                    } else {
+                        try {
+                            Files.copy(dir, toDestination(dir), copyOptions);
+                            System.out.println("Extracted " + dir);
+                        } catch (FileAlreadyExistsException ignore) {
+                        } catch (IOException x) {
+                            return FileVisitResult.SKIP_SUBTREE;
+                        }
                     }
+                    return FileVisitResult.CONTINUE;
                 }
-                return FileVisitResult.CONTINUE;
-            }
 
-            @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                Files.copy(file, target.resolve(source.relativize(file)), copyOptions);
-                return FileVisitResult.CONTINUE;
-            }
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    Files.copy(file, toDestination(file), copyOptions);
+                    System.out.println("Extracted " + file);
+                    return FileVisitResult.CONTINUE;
+                }
 
-            @Override
-            public FileVisitResult visitFileFailed(Path file, IOException exc) {
-                return FileVisitResult.CONTINUE;
-            }
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
 
-            @Override
-            public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
-                return FileVisitResult.CONTINUE;
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                private Path toDestination(final Path path) {
+                    final Path tmp = path.toAbsolutePath();
+                    return target.resolve(tmp.toString().substring(sourceLen));
+                }
+            });
+        } finally {
+            if (fs != null) {
+                fs.close();
             }
-        });
+        }
     }
 
 }

core/src/main/java/com/questdb/misc/ByteBuffers.java

@@ -222,7 +222,6 @@ public final class ByteBuffers {
         b.position(p);
     }
 
-
     public static long getAddress(ByteBuffer buffer) {
         return ((DirectBuffer) buffer).address();
     }
@@ -262,6 +261,10 @@ public final class ByteBuffers {
         return channelSize > max ? max : channelSize;
     }
 
+    public static boolean isDirect(ByteBuffer buf) {
+        return buf instanceof DirectBuffer;
+    }
+
     public static void putStr(ByteBuffer buffer, CharSequence value) {
         int p = buffer.position();
         for (int i = 0; i < value.length(); i++) {

core/src/main/java/com/questdb/net/GZipWritableChannel.java

@@ -26,7 +26,6 @@ package com.questdb.net;
 import com.questdb.misc.ByteBuffers;
 import com.questdb.misc.Unsafe;
 import com.questdb.misc.Zip;
-import sun.nio.ch.DirectBuffer;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -98,7 +97,7 @@ public class GZipWritableChannel<T extends WritableByteChannel> implements Writa
 
     @Override
     public int write(ByteBuffer src) throws IOException {
-        if (!(src instanceof DirectBuffer)) {
+        if (!ByteBuffers.isDirect(src)) {
             throw new IllegalArgumentException("Heap buffers are not supported");
         }
 

core/src/main/java/com/questdb/net/http/HttpServer.java

@@ -89,7 +89,7 @@ public class HttpServer {
         this.clock = clock;
     }
 
-    public void start(ObjHashSet<? extends Job> extraJobs, int queueDepth) {
+    public boolean start(ObjHashSet<? extends Job> extraJobs, int queueDepth) {
         this.running = true;
         ioQueue = new RingQueue<>(IOEvent.FACTORY, queueDepth);
         SPSequence ioPubSequence = new SPSequence(ioQueue.getCapacity());
@@ -102,7 +102,7 @@ public class HttpServer {
         } catch (NetworkError e) {
             LOG.error().$("Server failed to start: ").$(e.getMessage()).$();
             running = false;
-            return;
+            return false;
         }
 
         IOHttpJob ioHttp = new IOHttpJob(ioQueue, ioSubSequence, this.dispatcher, urlMatcher);
@@ -121,6 +121,7 @@ public class HttpServer {
         }
 
         startComplete.countDown();
+        return true;
     }
 
     public void start() {

core/src/main/java/com/questdb/ql/impl/analytic/denserank/DenseRankAnalyticFunctionFactory.java

@@ -49,7 +49,7 @@ public class DenseRankAnalyticFunctionFactory implements AnalyticFunctionFactory
         }
 
         if (ordered) {
-            return new DenseRankOrderedPartitionedAnalyticFunction(configuration.getDbAnalyticFuncPage(), valueColumnAlias, partitionBy);
+            return new DenseRankOPAnalyticFunction(configuration.getDbAnalyticFuncPage(), valueColumnAlias, partitionBy);
         }
 
         if (partitionBy != null) {

core/src/main/java/com/questdb/ql/impl/analytic/denserank/DenseRankOrderedPartitionedAnalyticFunction.java → core/src/main/java/com/questdb/ql/impl/analytic/denserank/DenseRankOPAnalyticFunction.java

@@ -33,11 +33,11 @@ import com.questdb.std.ObjList;
 
 import java.io.IOException;
 
-public class DenseRankOrderedPartitionedAnalyticFunction extends AbstractRankOrderedAnalyticFunction {
+public class DenseRankOPAnalyticFunction extends AbstractRankOrderedAnalyticFunction {
     private final DirectMap partitionMap;
     private final ObjList<VirtualColumn> partitionBy;
 
-    public DenseRankOrderedPartitionedAnalyticFunction(int pageSize, String name, ObjList<VirtualColumn> partitionBy) {
+    public DenseRankOPAnalyticFunction(int pageSize, String name, ObjList<VirtualColumn> partitionBy) {
         super(pageSize, name);
         this.partitionMap = new DirectMap(pageSize, 1, MapUtils.ROWID_MAP_VALUES);
         this.partitionBy = partitionBy;