From 699e1b5edee6ad913b7e9a93b6b3863f77cf7e75 Mon Sep 17 00:00:00 2001 From: Richard He Date: Mon, 9 Sep 2019 18:11:02 -0700 Subject: [PATCH] Added support for SstFileReader JNI interface (#5556) Summary: Feature request as per https://github.com/facebook/rocksdb/issues/5538 issue. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5556 Differential Revision: D17219008 fbshipit-source-id: e31f18dec318416eac9dea8213bab31da96e1f3a --- java/CMakeLists.txt | 6 + java/Makefile | 3 + java/rocksjni/sst_file_reader_iterator.cc | 186 ++++++++++++++++++ java/rocksjni/sst_file_readerjni.cc | 110 +++++++++++ .../main/java/org/rocksdb/SstFileReader.java | 83 ++++++++ .../org/rocksdb/SstFileReaderIterator.java | 65 ++++++ .../java/org/rocksdb/SstFileReaderTest.java | 137 +++++++++++++ src.mk | 2 + 8 files changed, 592 insertions(+) create mode 100644 java/rocksjni/sst_file_reader_iterator.cc create mode 100644 java/rocksjni/sst_file_readerjni.cc create mode 100644 java/src/main/java/org/rocksdb/SstFileReader.java create mode 100644 java/src/main/java/org/rocksdb/SstFileReaderIterator.java create mode 100644 java/src/test/java/org/rocksdb/SstFileReaderTest.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index f00b6f7f9..b1f706c16 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -51,6 +51,8 @@ set(JNI_NATIVE_SOURCES rocksjni/snapshot.cc rocksjni/sst_file_manager.cc rocksjni/sst_file_writerjni.cc + rocksjni/sst_file_readerjni.cc + rocksjni/sst_file_reader_iterator.cc rocksjni/statistics.cc rocksjni/statisticsjni.cc rocksjni/table.cc @@ -194,6 +196,8 @@ set(JAVA_MAIN_CLASSES src/main/java/org/rocksdb/SstFileManager.java src/main/java/org/rocksdb/SstFileMetaData.java src/main/java/org/rocksdb/SstFileWriter.java + src/main/java/org/rocksdb/SstFileReader.java + src/main/java/org/rocksdb/SstFileReaderIterator.java src/main/java/org/rocksdb/StateType.java src/main/java/org/rocksdb/StatisticsCollectorCallback.java src/main/java/org/rocksdb/StatisticsCollector.java @@ -436,6 +440,8 @@ if(${CMAKE_VERSION} VERSION_LESS "3.11.4" OR (${Java_VERSION_MINOR} STREQUAL "7" org.rocksdb.Snapshot org.rocksdb.SstFileManager org.rocksdb.SstFileWriter + org.rocksdb.SstFileReader + org.rocksdb.SstFileReaderIterator org.rocksdb.Statistics org.rocksdb.StringAppendOperator org.rocksdb.TableFormatConfig diff --git a/java/Makefile b/java/Makefile index 7aa15bfd0..f8642c2d6 100644 --- a/java/Makefile +++ b/java/Makefile @@ -60,6 +60,8 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\ org.rocksdb.Slice\ org.rocksdb.SstFileManager\ org.rocksdb.SstFileWriter\ + org.rocksdb.SstFileReader\ + org.rocksdb.SstFileReaderIterator\ org.rocksdb.Statistics\ org.rocksdb.ThreadStatus\ org.rocksdb.TimedEnv\ @@ -156,6 +158,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\ org.rocksdb.SnapshotTest\ org.rocksdb.SstFileManagerTest\ org.rocksdb.SstFileWriterTest\ + org.rocksdb.SstFileReaderTest\ org.rocksdb.TableFilterTest\ org.rocksdb.TimedEnvTest\ org.rocksdb.TransactionTest\ diff --git a/java/rocksjni/sst_file_reader_iterator.cc b/java/rocksjni/sst_file_reader_iterator.cc new file mode 100644 index 000000000..7ab97e412 --- /dev/null +++ b/java/rocksjni/sst_file_reader_iterator.cc @@ -0,0 +1,186 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ rocksdb::Iterator methods from Java side. + +#include +#include +#include + +#include "include/org_rocksdb_SstFileReaderIterator.h" +#include "rocksjni/portal.h" +#include "rocksdb/iterator.h" + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + assert(it != nullptr); + delete it; +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: isValid0 + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast(handle)->Valid(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekToFirst0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToFirst(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekToLast0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToLast(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: next0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Next(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: prev0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Prev(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seek0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/, + jlong handle, jbyteArray jtarget, + jint jtarget_len) { + jbyte* target = env->GetByteArrayElements(jtarget, nullptr); + if(target == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + rocksdb::Slice target_slice( + reinterpret_cast(target), jtarget_len); + + auto* it = reinterpret_cast(handle); + it->Seek(target_slice); + + env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekForPrev0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, + jlong handle, + jbyteArray jtarget, + jint jtarget_len) { + jbyte* target = env->GetByteArrayElements(jtarget, nullptr); + if(target == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + rocksdb::Slice target_slice( + reinterpret_cast(target), jtarget_len); + + auto* it = reinterpret_cast(handle); + it->SeekForPrev(target_slice); + + env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: status0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + rocksdb::Status s = it->status(); + + if (s.ok()) { + return; + } + + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: key0 + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_SstFileReaderIterator_key0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + rocksdb::Slice key_slice = it->key(); + + jbyteArray jkey = env->NewByteArray(static_cast(key_slice.size())); + if(jkey == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion(jkey, 0, static_cast(key_slice.size()), + const_cast(reinterpret_cast(key_slice.data()))); + return jkey; +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: value0 + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_SstFileReaderIterator_value0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + rocksdb::Slice value_slice = it->value(); + + jbyteArray jkeyValue = + env->NewByteArray(static_cast(value_slice.size())); + if(jkeyValue == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion(jkeyValue, 0, static_cast(value_slice.size()), + const_cast(reinterpret_cast(value_slice.data()))); + return jkeyValue; +} \ No newline at end of file diff --git a/java/rocksjni/sst_file_readerjni.cc b/java/rocksjni/sst_file_readerjni.cc new file mode 100644 index 000000000..32b914a2d --- /dev/null +++ b/java/rocksjni/sst_file_readerjni.cc @@ -0,0 +1,110 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ rocksdb::SstFileReader methods +// from Java side. + +#include +#include + +#include "include/org_rocksdb_SstFileReader.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/sst_file_reader.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_SstFileReader + * Method: newSstFileReader + * Signature: (J)J + */ +jlong Java_org_rocksdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/, + jclass /*jcls*/, + jlong joptions) { + auto *options = reinterpret_cast(joptions); + rocksdb::SstFileReader *sst_file_reader = + new rocksdb::SstFileReader(*options); + return reinterpret_cast(sst_file_reader); +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: open + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_SstFileReader_open(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, jstring jfile_path) { + const char *file_path = env->GetStringUTFChars(jfile_path, nullptr); + if (file_path == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + rocksdb::Status s = + reinterpret_cast(jhandle)->Open(file_path); + env->ReleaseStringUTFChars(jfile_path, file_path); + + if (!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* +* Class: org_rocksdb_SstFileReader +* Method: newIterator +* Signature: (JJ)J +*/ +jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jread_options_handle) { + auto* sst_file_reader = reinterpret_cast(jhandle); + auto* read_options = + reinterpret_cast(jread_options_handle); + return reinterpret_cast(sst_file_reader->NewIterator(*read_options)); +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReader_disposeInternal(JNIEnv * /*env*/, + jobject /*jobj*/, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: verifyChecksum + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env, + jobject /*jobj*/, + jlong jhandle) { + auto* sst_file_reader = reinterpret_cast(jhandle); + auto s = sst_file_reader->VerifyChecksum(); + if (!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: getTableProperties + * Signature: (J)J + */ +jobject Java_org_rocksdb_SstFileReader_getTableProperties(JNIEnv *env, + jobject /*jobj*/, + jlong jhandle) { + auto* sst_file_reader = reinterpret_cast(jhandle); + std::shared_ptr tp = sst_file_reader->GetTableProperties(); + jobject jtable_properties = rocksdb::TablePropertiesJni::fromCppTableProperties( + env, *(tp.get())); + return jtable_properties; +} + diff --git a/java/src/main/java/org/rocksdb/SstFileReader.java b/java/src/main/java/org/rocksdb/SstFileReader.java new file mode 100644 index 000000000..66349f32f --- /dev/null +++ b/java/src/main/java/org/rocksdb/SstFileReader.java @@ -0,0 +1,83 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class SstFileReader extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + public SstFileReader(final Options options) { + super(newSstFileReader(options.nativeHandle_)); + } + + /** + * Returns an iterator that will iterate on all keys in the default + * column family including both keys in the DB and uncommitted keys in this + * transaction. + * + * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is read + * from the DB but will NOT change which keys are read from this transaction + * (the keys in this transaction do not yet belong to any snapshot and will be + * fetched regardless). + * + * Caller is responsible for deleting the returned Iterator. + * + * @param readOptions Read options. + * + * @return instance of iterator object. + */ + public SstFileReaderIterator newIterator(final ReadOptions readOptions) { + assert(isOwningHandle()); + long iter = newIterator(nativeHandle_, + readOptions.nativeHandle_); + return new SstFileReaderIterator(this, iter); + } + + /** + * Prepare SstFileReader to read a file. + * + * @param filePath the location of file + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void open(final String filePath) throws RocksDBException { + open(nativeHandle_, filePath); + } + + /** + * Verify checksum + * + * @throws RocksDBException if the checksum is not valid + */ + public void verifyChecksum() throws RocksDBException { + verifyChecksum(nativeHandle_); + } + + /** + * Get the properties of the table. + * + * + * @return the properties + */ + public TableProperties getTableProperties() throws RocksDBException { + return getTableProperties(nativeHandle_); + } + + + + @Override protected final native void disposeInternal(final long handle); + private native long newIterator(final long handle, + final long readOptionsHandle); + + private native void open(final long handle, final String filePath) + throws RocksDBException; + + private native static long newSstFileReader(final long optionsHandle); + private native void verifyChecksum(final long handle) throws RocksDBException; + private native TableProperties getTableProperties(final long handle) throws RocksDBException; +} diff --git a/java/src/main/java/org/rocksdb/SstFileReaderIterator.java b/java/src/main/java/org/rocksdb/SstFileReaderIterator.java new file mode 100644 index 000000000..ef303f042 --- /dev/null +++ b/java/src/main/java/org/rocksdb/SstFileReaderIterator.java @@ -0,0 +1,65 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + *

An iterator that yields a sequence of key/value pairs from a source. + * Multiple implementations are provided by this library. + * In particular, iterators are provided + * to access the contents of a Table or a DB.

+ * + *

Multiple threads can invoke const methods on an RocksIterator without + * external synchronization, but if any of the threads may call a + * non-const method, all threads accessing the same RocksIterator must use + * external synchronization.

+ * + * @see RocksObject + */ +public class SstFileReaderIterator extends AbstractRocksIterator { + protected SstFileReaderIterator(SstFileReader reader, long nativeHandle) { + super(reader, nativeHandle); + } + + /** + *

Return the key for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

+ * + *

REQUIRES: {@link #isValid()}

+ * + * @return key for the current entry. + */ + public byte[] key() { + assert(isOwningHandle()); + return key0(nativeHandle_); + } + + /** + *

Return the value for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

+ * + *

REQUIRES: !AtEnd() && !AtStart()

+ * @return value for the current entry. + */ + public byte[] value() { + assert(isOwningHandle()); + return value0(nativeHandle_); + } + + @Override protected final native void disposeInternal(final long handle); + @Override final native boolean isValid0(long handle); + @Override final native void seekToFirst0(long handle); + @Override final native void seekToLast0(long handle); + @Override final native void next0(long handle); + @Override final native void prev0(long handle); + @Override final native void seek0(long handle, byte[] target, int targetLen); + @Override final native void seekForPrev0(long handle, byte[] target, int targetLen); + @Override final native void status0(long handle) throws RocksDBException; + + private native byte[] key0(long handle); + private native byte[] value0(long handle); +} diff --git a/java/src/test/java/org/rocksdb/SstFileReaderTest.java b/java/src/test/java/org/rocksdb/SstFileReaderTest.java new file mode 100644 index 000000000..5ccc8dec9 --- /dev/null +++ b/java/src/test/java/org/rocksdb/SstFileReaderTest.java @@ -0,0 +1,137 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.BytewiseComparator; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class SstFileReaderTest { + private static final String SST_FILE_NAME = "test.sst"; + + class KeyValueWithOp { + KeyValueWithOp(String key, String value, OpType opType) { + this.key = key; + this.value = value; + this.opType = opType; + } + + String getKey() { + return key; + } + + String getValue() { + return value; + } + + OpType getOpType() { + return opType; + } + + private String key; + private String value; + private OpType opType; + } + + @Rule public TemporaryFolder parentFolder = new TemporaryFolder(); + + enum OpType { PUT, PUT_BYTES, MERGE, MERGE_BYTES, DELETE, DELETE_BYTES} + + private File newSstFile(final List keyValues) throws IOException, RocksDBException { + final EnvOptions envOptions = new EnvOptions(); + final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options options = new Options().setMergeOperator(stringAppendOperator); + SstFileWriter sstFileWriter; + sstFileWriter = new SstFileWriter(envOptions, options); + + final File sstFile = parentFolder.newFile(SST_FILE_NAME); + try { + sstFileWriter.open(sstFile.getAbsolutePath()); + for (KeyValueWithOp keyValue : keyValues) { + Slice keySlice = new Slice(keyValue.getKey()); + Slice valueSlice = new Slice(keyValue.getValue()); + byte[] keyBytes = keyValue.getKey().getBytes(); + byte[] valueBytes = keyValue.getValue().getBytes(); + switch (keyValue.getOpType()) { + case PUT: + sstFileWriter.put(keySlice, valueSlice); + break; + case PUT_BYTES: + sstFileWriter.put(keyBytes, valueBytes); + break; + case MERGE: + sstFileWriter.merge(keySlice, valueSlice); + break; + case MERGE_BYTES: + sstFileWriter.merge(keyBytes, valueBytes); + break; + case DELETE: + sstFileWriter.delete(keySlice); + break; + case DELETE_BYTES: + sstFileWriter.delete(keyBytes); + break; + default: + fail("Unsupported op type"); + } + keySlice.close(); + valueSlice.close(); + } + sstFileWriter.finish(); + } finally { + assertThat(sstFileWriter).isNotNull(); + sstFileWriter.close(); + options.close(); + envOptions.close(); + } + return sstFile; + } + + @Test + public void readSstFile() throws RocksDBException, IOException { + final List keyValues = new ArrayList<>(); + keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT)); + + + final File sstFile = newSstFile(keyValues); + try(final StringAppendOperator stringAppendOperator = + new StringAppendOperator(); + final Options options = new Options() + .setCreateIfMissing(true) + .setMergeOperator(stringAppendOperator); + final SstFileReader reader = new SstFileReader(options) + ) { + // Open the sst file and iterator + reader.open(sstFile.getAbsolutePath()); + final ReadOptions readOptions = new ReadOptions(); + final SstFileReaderIterator iterator = reader.newIterator(readOptions); + + // Use the iterator to read sst file + iterator.seekToFirst(); + + // Verify Checksum + reader.verifyChecksum(); + + // Verify Table Properties + assertEquals(reader.getTableProperties().getNumEntries(), 1); + + // Check key and value + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + } + } + +} diff --git a/src.mk b/src.mk index 4c7689755..e37c8c919 100644 --- a/src.mk +++ b/src.mk @@ -476,6 +476,8 @@ JNI_NATIVE_SOURCES = \ java/rocksjni/snapshot.cc \ java/rocksjni/sst_file_manager.cc \ java/rocksjni/sst_file_writerjni.cc \ + java/rocksjni/sst_file_readerjni.cc \ + java/rocksjni/sst_file_reader_iterator.cc \ java/rocksjni/statistics.cc \ java/rocksjni/statisticsjni.cc \ java/rocksjni/table.cc \ -- GitLab