Merge pull request #318 from criccomini/master

Build RocksDB JNI cross-platform fat jar

.gitignore

@@ -34,4 +34,7 @@ tags
 java/*.log
 java/include/org_rocksdb_*.h
 unity.cc
+java/crossbuild/.vagrant
 .vagrant/
+java/**.asc
+java/javadoc

Makefile

@@ -164,6 +164,10 @@ endif
 LIBRARY = ${LIBNAME}.a
 MEMENVLIBRARY = libmemenv.a
 
+ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+ROCKSDB_PATCH = $(shell egrep "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+
 default: all
 
 #-----------------------------------------------
@@ -273,6 +277,7 @@ clean:
 	-rm -rf ios-x86/* ios-arm/*
 	-find . -name "*.[oda]" -exec rm {} \;
 	-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
+	-rm -rf bzip2* snappy* zlib*
 tags:
 	ctags * -R
 	cscope -b `find . -name '*.cc'` `find . -name '*.h'`
@@ -513,10 +518,16 @@ ldb: tools/ldb.o $(LIBOBJECTS)
 
 JNI_NATIVE_SOURCES = ./java/rocksjni/*.cc
 JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
-ROCKSDBJNILIB = librocksdbjni.so
-ROCKSDB_JAR = rocksdbjni.jar
+ARCH := $(shell getconf LONG_BIT)
+ROCKSDBJNILIB = librocksdbjni-linux$(ARCH).so
+ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux$(ARCH).jar
+ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
+ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar
+ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
 
 ifeq ($(PLATFORM), OS_MACOSX)
+ROCKSDBJNILIB = librocksdbjni-osx.jnilib
+ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
 JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
 endif
 
@@ -549,7 +560,20 @@ rocksdbjavastatic: libz.a libbz2.a libsnappy.a
 	rm -f ./java/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(LIBOBJECTS) $(COVERAGEFLAGS) libz.a libbz2.a libsnappy.a
 	cd java;jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md $(ROCKSDBJNILIB)
-
+	cd java/javadoc;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
+	cd java;jar -cf $(ROCKSDB_SOURCES_JAR) org
+
+rocksdbjavastaticrelease: rocksdbjavastatic
+	cd java/crossbuild && vagrant destroy -f && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64
+	cd java;jar -cf $(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md librocksdbjni-*.so librocksdbjni-*.jnilib
+
+rocksdbjavastaticpublish: rocksdbjavastaticrelease
+	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar -Dclassifier=javadoc
+	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar -Dclassifier=sources
+	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux64.jar -Dclassifier=linux64
+	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux32.jar -Dclassifier=linux32
+	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar -Dclassifier=osx
+	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
 
 rocksdbjava:
 	OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32

java/Makefile

 NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.BlockBasedTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB org.rocksdb.RocksEnv org.rocksdb.GenericRateLimiterConfig
 
+ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+ROCKSDB_PATCH = $(shell egrep "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+
 NATIVE_INCLUDE = ./include
-ROCKSDB_JAR = rocksdbjni.jar
+ARCH := $(shell getconf LONG_BIT)
+ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux$(ARCH).jar
+
+ifeq ($(PLATFORM), OS_MACOSX)
+ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
+endif
 
 clean:
 	-find . -name "*.class" -exec rm {} \;
 	-find . -name "hs*.log" -exec rm {} \;
-	rm -f $(ROCKSDB_JAR)
+	rm -rf javadoc/*
+
+javadocs:
+	mkdir -p javadoc; javadoc -d javadoc -sourcepath . -subpackages org
 
-java:
+java: javadocs
 	javac org/rocksdb/util/*.java org/rocksdb/*.java
 	@cp ../HISTORY.md ./HISTORY-CPP.md
 	@rm -f ./HISTORY-CPP.md

java/RELEASE.md

+## Cross-building
+
+RocksDB can be built as a single self contained cross-platform JAR. The cross-platform jar can be usd on any 64-bit OSX system, 32-bit Linux system, or 64-bit Linux system.
+
+Building a cross-platform JAR requires:
+
+ * [Vagrant](https://www.vagrantup.com/)
+ * [Virtualbox](https://www.virtualbox.org/)
+ * A Mac OSX machine that can compile RocksDB.
+ * Java 7 set as JAVA_HOME.
+
+Once you have these items, run this make command from RocksDB's root source directory:
+
+    make jclean clean rocksdbjavastaticrelease
+
+This command will build RocksDB natively on OSX, and will then spin up two Vagrant Virtualbox Ubuntu images to build RocksDB for both 32-bit and 64-bit Linux. 
+
+You can find all native binaries and JARs in the java directory upon completion:
+
+    librocksdbjni-linux32.so
+    librocksdbjni-linux64.so
+    librocksdbjni-osx.jnilib
+    rocksdbjni-3.5.0-javadoc.jar
+    rocksdbjni-3.5.0-linux32.jar
+    rocksdbjni-3.5.0-linux64.jar
+    rocksdbjni-3.5.0-osx.jar
+    rocksdbjni-3.5.0-sources.jar
+    rocksdbjni-3.5.0.jar
+
+## Maven publication
+
+Set ~/.m2/settings.xml to contain:
+
+    <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+      <servers>
+        <server>
+          <id>sonatype-nexus-staging</id>
+          <username>your-sonatype-jira-username</username>
+          <password>your-sonatype-jira-password</password>
+        </server>
+      </servers>
+    </settings>
+
+Then update rocksjni.pom's version tag to reflect the release version.
+
+From RocksDB's root directory, first build the Java static JARs:
+
+    make jclean clean rocksdbjavastaticpublish
+
+This command will [stage the JAR artifacts on the Sonatype staging repository](http://central.sonatype.org/pages/manual-staging-bundle-creation-and-deployment.html). To release the staged artifacts.
+
+1. Go to [https://oss.sonatype.org/#stagingRepositories](https://oss.sonatype.org/#stagingRepositories) and search for "rocksdb" in the upper right hand search box.
+2. Select the rocksdb staging repository, and inspect its contents.
+3. If all is well, follow [these steps](https://oss.sonatype.org/#stagingRepositories) to close the repository and release it.
+
+After the release has occurred, the artifacts will be synced to Maven central within 24-48 hours.

java/crossbuild/Vagrantfile

+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  config.vm.define "linux32" do |linux32|
+    linux32.vm.box = "hansode/centos-5.6-i386"
+  end
+
+  config.vm.define "linux64" do |linux64|
+    linux64.vm.box = "hansode/centos-5.6-x86_64"
+  end
+
+  config.vm.provider "virtualbox" do |v|
+    v.memory = 2048
+    v.cpus = 4
+  end
+
+  config.vm.provision :shell, path: "build-linux-centos.sh"
+  config.vm.synced_folder "../", "/rocksdb-build"
+  config.vm.synced_folder "../..", "/rocksdb", type: "rsync"
+end

java/crossbuild/build-linux-centos.sh

+#!/usr/bin/env bash
+# install all required packages for rocksdb that are available through yum
+ARCH=$(uname -i)
+sudo yum -y install java-1.6.0-openjdk-devel.$ARCH zlib zlib-devel bzip2 bzip2-devel
+
+# install gcc/g++ 4.7 via CERN (http://linux.web.cern.ch/linux/devtoolset/)
+sudo wget -O /etc/yum.repos.d/slc5-devtoolset.repo http://linuxsoft.cern.ch/cern/devtoolset/slc5-devtoolset.repo
+sudo wget -O /etc/pki/rpm-gpg/RPM-GPG-KEY-cern http://ftp.mirrorservice.org/sites/ftp.scientificlinux.org/linux/scientific/51/i386/RPM-GPG-KEYs/RPM-GPG-KEY-cern
+sudo yum -y install devtoolset-1.1 
+wget http://gflags.googlecode.com/files/gflags-1.6.tar.gz
+tar xvfz gflags-1.6.tar.gz; cd gflags-1.6; scl enable devtoolset-1.1 ./configure; scl enable devtoolset-1.1 make; sudo make install
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
+# set java home so we can build rocksdb jars
+export JAVA_HOME=/usr/lib/jvm/java-1.6.0
+
+# build rocksdb
+cd /rocksdb
+scl enable devtoolset-1.1 'make jclean clean'
+scl enable devtoolset-1.1 'make -j 4 rocksdbjavastatic'
+cp /rocksdb/java/librocksdbjni-* /rocksdb-build
+cp /rocksdb/java/rocksdbjni-* /rocksdb-build
+

java/crossbuild/build-linux.sh

+#!/usr/bin/env bash
+# install all required packages for rocksdb
+sudo apt-get update
+sudo apt-get -y install git make gcc g++ libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev default-jdk
+
+# set java home so we can build rocksdb jars
+export JAVA_HOME=$(echo /usr/lib/jvm/java-7-openjdk*)
+cd /rocksdb
+make jclean clean
+make -j 4 rocksdbjavastatic
+cp /rocksdb/java/librocksdbjni-* /rocksdb-build
+cp /rocksdb/java/rocksdbjni-* /rocksdb-build
+sudo shutdown -h now
+

java/org/rocksdb/NativeLibraryLoader.java

 package org.rocksdb;
 
 import java.io.*;
-
+import org.rocksdb.util.Environment;
 
 /**
  * This class is used to load the RocksDB shared library from within the jar.
  * The shared library is extracted to a temp folder and loaded from there.
  */
 public class NativeLibraryLoader {
-  private static String sharedLibraryName = "librocksdbjni.so";
+  private static String sharedLibraryName = Environment.getJniLibraryName("rocksdb");
   private static String tempFilePrefix = "librocksdbjni";
-  private static String tempFileSuffix = ".so";
+  private static String tempFileSuffix = "." + Environment.getJniLibraryExtension();
 
   public static void loadLibraryFromJar(String tmpDir)
       throws IOException {

java/org/rocksdb/util/Environment.java

@@ -2,6 +2,7 @@ package org.rocksdb.util;
 
 public class Environment {
   private static String OS = System.getProperty("os.name").toLowerCase();
+  private static String ARCH = System.getProperty("os.arch").toLowerCase();
 
   public static boolean isWindows() {
     return (OS.indexOf("win") >= 0);
@@ -17,21 +18,30 @@ public class Environment {
             OS.indexOf("aix") >= 0);
   }
 
+  public static boolean is64Bit() {
+    return (ARCH.indexOf("64") > 0);
+  }
+
   public static String getSharedLibraryName(String name) {
     if (isUnix()) {
-      return String.format("lib%s.so", name);
+      return String.format("lib%sjni.so", name);
     } else if (isMac()) {
-      return String.format("lib%s.dylib", name);
+      return String.format("lib%sjni.dylib", name);
     }
     throw new UnsupportedOperationException();
   }
 
   public static String getJniLibraryName(String name) {
     if (isUnix()) {
-      return String.format("lib%s.so", name);
+      String arch = (is64Bit()) ? "64" : "32";
+      return String.format("lib%sjni-linux%s.so", name, arch);
     } else if (isMac()) {
-      return String.format("lib%s.jnilib", name);
+      return String.format("lib%sjni-osx.jnilib", name);
     }
     throw new UnsupportedOperationException();
   }
+
+  public static String getJniLibraryExtension() {
+    return (isMac()) ? ".jnilib" : ".so";
+  }
 }

java/rocksjni.pom

+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <name>RocksDB JNI</name>
+  <url>http://rocksdb.org/</url>
+  <groupId>org.rocksdb</groupId>
+  <artifactId>rocksdbjni</artifactId>
+  <version>3.6.0</version>
+  <description>RocksDB fat jar that contains .so files for linux32 and linux64, and jnilib files for Mac OSX.</description>
+  <licenses>
+    <license>
+      <name>Apache License 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+  <scm>
+    <connection>scm:git:git://github.com/dropwizard/metrics.git</connection>
+    <developerConnection>scm:git:git@github.com:dropwizard/metrics.git</developerConnection>
+    <url>http://github.com/dropwizard/metrics/</url>
+    <tag>HEAD</tag>
+  </scm>
+  <developers>
+    <developer>
+      <name>Facebook</name>
+      <email>help@facebook.com</email>
+      <timezone>America/New_York</timezone>
+      <roles>
+        <role>architect</role>
+      </roles>
+    </developer>
+  </developers>
+</project>