提交 86cf7266 编写于 作者: A Alan Paxton 提交者: Facebook GitHub Bot

keyMayExist() supports ByteBuffer (#9013)

Summary:
closes https://github.com/facebook/rocksdb/issues/7917

Implemented ByteBuffer API variants of Java keyMayExist() uniformly with and without column families, read options and return data values. Implemented 2 supporting C++ JNI methods.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9013

Reviewed By: mrambacher

Differential Revision: D31665989

Pulled By: jay-zhuang

fbshipit-source-id: 8adc1730217dba38d6fa7b31d788650a33e28af1
上级 53a0ab2b
......@@ -16,6 +16,7 @@
* Add remote compaction read/write bytes statistics: `REMOTE_COMPACT_READ_BYTES`, `REMOTE_COMPACT_WRITE_BYTES`.
* Introduce an experimental feature to dump out the blocks from block cache and insert them to the secondary cache to reduce the cache warmup time (e.g., used while migrating DB instance). More information are in `class CacheDumper` and `CacheDumpedLoader` at `rocksdb/utilities/cache_dump_load.h` Note that, this feature is subject to the potential change in the future, it is still experimental.
* Introduced a new BlobDB configuration option `blob_garbage_collection_force_threshold`, which can be used to trigger compactions targeting the SST files which reference the oldest blob files when the ratio of garbage in those blob files meets or exceeds the specified threshold. This can reduce space amplification with skewed workloads where the affected SST files might not otherwise get picked up for compaction.
* [JAVA] `keyMayExist()` supports ByteBuffer.
### Public API change
* Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method.
......
......@@ -167,6 +167,7 @@ set(JAVA_MAIN_CLASSES
src/main/java/org/rocksdb/LevelMetaData.java
src/main/java/org/rocksdb/ConcurrentTaskLimiter.java
src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java
src/main/java/org/rocksdb/KeyMayExist.java
src/main/java/org/rocksdb/LiveFileMetaData.java
src/main/java/org/rocksdb/LogFile.java
src/main/java/org/rocksdb/Logger.java
......
......@@ -40,6 +40,7 @@ NATIVE_JAVA_CLASSES = \
org.rocksdb.HdfsEnv\
org.rocksdb.ConcurrentTaskLimiter\
org.rocksdb.ConcurrentTaskLimiterImpl\
org.rocksdb.KeyMayExist\
org.rocksdb.Logger\
org.rocksdb.LRUCache\
org.rocksdb.MemoryUsageType\
......
......@@ -1929,6 +1929,49 @@ bool key_may_exist_helper(JNIEnv* env, jlong jdb_handle, jlong jcf_handle,
return exists;
}
bool key_may_exist_direct_helper(JNIEnv* env, jlong jdb_handle,
jlong jcf_handle, jlong jread_opts_handle,
jobject jkey, jint jkey_offset, jint jkey_len,
bool* has_exception, std::string* value,
bool* value_found) {
auto* db = reinterpret_cast<ROCKSDB_NAMESPACE::DB*>(jdb_handle);
ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle;
if (jcf_handle == 0) {
cf_handle = db->DefaultColumnFamily();
} else {
cf_handle =
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyHandle*>(jcf_handle);
}
ROCKSDB_NAMESPACE::ReadOptions read_opts =
jread_opts_handle == 0
? ROCKSDB_NAMESPACE::ReadOptions()
: *(reinterpret_cast<ROCKSDB_NAMESPACE::ReadOptions*>(
jread_opts_handle));
char* key = reinterpret_cast<char*>(env->GetDirectBufferAddress(jkey));
if (key == nullptr) {
ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(
env,
"Invalid key argument (argument is not a valid direct ByteBuffer)");
*has_exception = true;
return false;
}
if (env->GetDirectBufferCapacity(jkey) < (jkey_offset + jkey_len)) {
ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(
env,
"Invalid key argument. Capacity is less than requested region (offset "
"+ length).");
*has_exception = true;
return false;
}
ROCKSDB_NAMESPACE::Slice key_slice(key, jkey_len);
const bool exists =
db->KeyMayExist(read_opts, cf_handle, key_slice, value, value_found);
return exists;
}
/*
* Class: org_rocksdb_RocksDB
......@@ -1957,6 +2000,101 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist(
return static_cast<jboolean>(exists);
}
/*
* Class: org_rocksdb_RocksDB
* Method: keyMayExistDirect
* Signature: (JJJLjava/nio/ByteBuffer;II)Z
*/
jboolean Java_org_rocksdb_RocksDB_keyMayExistDirect(
JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle,
jlong jread_opts_handle, jobject jkey, jint jkey_offset, jint jkey_len) {
bool has_exception = false;
std::string value;
bool value_found = false;
const bool exists = key_may_exist_direct_helper(
env, jdb_handle, jcf_handle, jread_opts_handle, jkey, jkey_offset,
jkey_len, &has_exception, &value, &value_found);
if (has_exception) {
// java exception already raised
return false;
}
return static_cast<jboolean>(exists);
}
/*
* Class: org_rocksdb_RocksDB
* Method: keyMayExistDirectFoundValue
* Signature:
* (JJJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)[J
*/
jintArray Java_org_rocksdb_RocksDB_keyMayExistDirectFoundValue(
JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle,
jlong jread_opts_handle, jobject jkey, jint jkey_offset, jint jkey_len,
jobject jval, jint jval_offset, jint jval_len) {
char* val_buffer = reinterpret_cast<char*>(env->GetDirectBufferAddress(jval));
if (val_buffer == nullptr) {
ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(
env,
"Invalid value argument (argument is not a valid direct ByteBuffer)");
return nullptr;
}
if (env->GetDirectBufferCapacity(jval) < (jval_offset + jval_len)) {
ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(
env,
"Invalid value argument. Capacity is less than requested region "
"(offset + length).");
return nullptr;
}
bool has_exception = false;
std::string cvalue;
bool value_found = false;
const bool exists = key_may_exist_direct_helper(
env, jdb_handle, jcf_handle, jread_opts_handle, jkey, jkey_offset,
jkey_len, &has_exception, &cvalue, &value_found);
if (has_exception) {
// java exception already raised
return nullptr;
}
const jint cvalue_len = static_cast<jint>(cvalue.size());
const jint length = std::min(jval_len, cvalue_len);
memcpy(val_buffer + jval_offset, cvalue.c_str(), length);
// keep consistent with java KeyMayExistEnum.values()
const int kNotExist = 0;
const int kExistsWithoutValue = 1;
const int kExistsWithValue = 2;
// TODO fix return value/type
// exists/value_found/neither
// cvalue_len
jintArray jresult = env->NewIntArray(2);
const jint jexists =
exists ? (value_found ? kExistsWithValue : kExistsWithoutValue)
: kNotExist;
env->SetIntArrayRegion(jresult, 0, 1, &jexists);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
env->DeleteLocalRef(jresult);
return nullptr;
}
env->SetIntArrayRegion(jresult, 1, 1, &cvalue_len);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
env->DeleteLocalRef(jresult);
return nullptr;
}
return jresult;
}
/*
* Class: org_rocksdb_RocksDB
* Method: keyMayExistFoundValue
......@@ -1964,17 +2102,14 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist(
*/
jobjectArray Java_org_rocksdb_RocksDB_keyMayExistFoundValue(
JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle,
jlong jread_opts_handle,
jbyteArray jkey, jint jkey_offset, jint jkey_len) {
jlong jread_opts_handle, jbyteArray jkey, jint jkey_offset, jint jkey_len) {
bool has_exception = false;
std::string value;
bool value_found = false;
const bool exists = key_may_exist_helper(
env, jdb_handle, jcf_handle, jread_opts_handle,
jkey, jkey_offset, jkey_len,
&has_exception, &value, &value_found);
env, jdb_handle, jcf_handle, jread_opts_handle, jkey, jkey_offset,
jkey_len, &has_exception, &value, &value_found);
if (has_exception) {
// java exception already raised
......@@ -2009,12 +2144,12 @@ jobjectArray Java_org_rocksdb_RocksDB_keyMayExistFoundValue(
env->DeleteLocalRef(jresult_flags);
return nullptr;
}
env->SetObjectArrayElement(jresults, 0, jresult_flags);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
env->DeleteLocalRef(jresult_flags);
return nullptr;
// exception thrown: ArrayIndexOutOfBoundsException
env->DeleteLocalRef(jresult_flags);
return nullptr;
}
env->DeleteLocalRef(jresult_flags);
......
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
package org.rocksdb;
import java.util.Objects;
public class KeyMayExist {
@Override
public boolean equals(final Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
final KeyMayExist that = (KeyMayExist) o;
return (valueLength == that.valueLength && exists == that.exists);
}
@Override
public int hashCode() {
return Objects.hash(exists, valueLength);
}
enum KeyMayExistEnum { kNotExist, kExistsWithoutValue, kExistsWithValue }
;
public KeyMayExist(final KeyMayExistEnum exists, final int valueLength) {
this.exists = exists;
this.valueLength = valueLength;
}
final KeyMayExistEnum exists;
final int valueLength;
}
......@@ -2218,8 +2218,8 @@ public class RocksDB extends RocksObject {
assert(keys.size() != 0);
final byte[][] keysArray = keys.toArray(new byte[0][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2278,8 +2278,8 @@ public class RocksDB extends RocksObject {
}
final byte[][] keysArray = keys.toArray(new byte[0][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2317,8 +2317,8 @@ public class RocksDB extends RocksObject {
assert(keys.size() != 0);
final byte[][] keysArray = keys.toArray(new byte[0][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2378,8 +2378,8 @@ public class RocksDB extends RocksObject {
}
final byte[][] keysArray = keys.toArray(new byte[0][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2415,8 +2415,8 @@ public class RocksDB extends RocksObject {
assert(keys.size() != 0);
final byte[][] keysArray = keys.toArray(new byte[keys.size()][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2461,8 +2461,8 @@ public class RocksDB extends RocksObject {
}
final byte[][] keysArray = keys.toArray(new byte[keys.size()][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2488,8 +2488,8 @@ public class RocksDB extends RocksObject {
assert(keys.size() != 0);
final byte[][] keysArray = keys.toArray(new byte[keys.size()][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2534,8 +2534,8 @@ public class RocksDB extends RocksObject {
}
final byte[][] keysArray = keys.toArray(new byte[keys.size()][]);
final int keyOffsets[] = new int[keysArray.length];
final int keyLengths[] = new int[keysArray.length];
final int[] keyOffsets = new int[keysArray.length];
final int[] keyLengths = new int[keysArray.length];
for(int i = 0; i < keyLengths.length; i++) {
keyLengths[i] = keysArray[i].length;
}
......@@ -2548,7 +2548,7 @@ public class RocksDB extends RocksObject {
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a true negative.
* positives, but never a false negative.
*
* If the caller wants to obtain value when the key
* is found in memory, then {@code valueHolder} must be set.
......@@ -2574,7 +2574,7 @@ public class RocksDB extends RocksObject {
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a true negative.
* positives, but never a false negative.
*
* If the caller wants to obtain value when the key
* is found in memory, then {@code valueHolder} must be set.
......@@ -2605,7 +2605,7 @@ public class RocksDB extends RocksObject {
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a true negative.
* positives, but never a false negative.
*
* If the caller wants to obtain value when the key
* is found in memory, then {@code valueHolder} must be set.
......@@ -2634,7 +2634,7 @@ public class RocksDB extends RocksObject {
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a true negative.
* positives, but never a false negative.
*
* If the caller wants to obtain value when the key
* is found in memory, then {@code valueHolder} must be set.
......@@ -2762,7 +2762,7 @@ public class RocksDB extends RocksObject {
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a true negative.
* positives, but never a false negative.
*
* If the caller wants to obtain value when the key
* is found in memory, then {@code valueHolder} must be set.
......@@ -2815,6 +2815,158 @@ public class RocksDB extends RocksObject {
}
}
/**
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a false negative.
*
* @param key bytebuffer containing the value of the key
* @return false if the key definitely does not exist in the database,
* otherwise true.
*/
public boolean keyMayExist(final ByteBuffer key) {
return keyMayExist(null, (ReadOptions) null, key);
}
/**
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a false negative.
*
* @param columnFamilyHandle the {@link ColumnFamilyHandle} to look for the key in
* @param key bytebuffer containing the value of the key
* @return false if the key definitely does not exist in the database,
* otherwise true.
*/
public boolean keyMayExist(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key) {
return keyMayExist(columnFamilyHandle, (ReadOptions) null, key);
}
/**
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a false negative.
*
* @param readOptions the {@link ReadOptions} to use when reading the key/value
* @param key bytebuffer containing the value of the key
* @return false if the key definitely does not exist in the database,
* otherwise true.
*/
public boolean keyMayExist(final ReadOptions readOptions, final ByteBuffer key) {
return keyMayExist(null, readOptions, key);
}
/**
* If the key definitely does not exist in the database, then this method
* returns {@link KeyMayExist.KeyMayExistEnum#kNotExist},
* otherwise if it can with best effort retreive the value, it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might
* exist is at the discretion of the implementation; the only guarantee is that {@link
* KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist.
*
* @param key bytebuffer containing the value of the key
* @param value bytebuffer which will receive a value if the key exists and a value is known
* @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided
*/
public KeyMayExist keyMayExist(final ByteBuffer key, final ByteBuffer value) {
return keyMayExist(null, null, key, value);
}
/**
* If the key definitely does not exist in the database, then this method
* returns {@link KeyMayExist.KeyMayExistEnum#kNotExist},
* otherwise if it can with best effort retreive the value, it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might
* exist is at the discretion of the implementation; the only guarantee is that {@link
* KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist.
*
* @param columnFamilyHandle the {@link ColumnFamilyHandle} to look for the key in
* @param key bytebuffer containing the value of the key
* @param value bytebuffer which will receive a value if the key exists and a value is known
* @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided
*/
public KeyMayExist keyMayExist(
final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, final ByteBuffer value) {
return keyMayExist(columnFamilyHandle, null, key, value);
}
/**
* If the key definitely does not exist in the database, then this method
* returns {@link KeyMayExist.KeyMayExistEnum#kNotExist},
* otherwise if it can with best effort retreive the value, it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might
* exist is at the discretion of the implementation; the only guarantee is that {@link
* KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist.
*
* @param readOptions the {@link ReadOptions} to use when reading the key/value
* @param key bytebuffer containing the value of the key
* @param value bytebuffer which will receive a value if the key exists and a value is known
* @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided
*/
public KeyMayExist keyMayExist(
final ReadOptions readOptions, final ByteBuffer key, final ByteBuffer value) {
return keyMayExist(null, readOptions, key, value);
}
/**
* If the key definitely does not exist in the database, then this method
* returns false, otherwise it returns true if the key might exist.
* That is to say that this method is probabilistic and may return false
* positives, but never a false negative.
*
* @param columnFamilyHandle
* @param readOptions
* @param key
* @return
*/
public boolean keyMayExist(final ColumnFamilyHandle columnFamilyHandle,
final ReadOptions readOptions, final ByteBuffer key) {
assert key != null : "key ByteBuffer parameter cannot be null";
assert key.isDirect() : "key parameter must be a direct ByteBuffer";
return keyMayExistDirect(nativeHandle_,
columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_,
readOptions == null ? 0 : readOptions.nativeHandle_, key, key.position(), key.limit());
}
/**
* If the key definitely does not exist in the database, then this method
* returns {@link KeyMayExist.KeyMayExistEnum#kNotExist},
* otherwise if it can with best effort retreive the value, it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link
* KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might
* exist is at the discretion of the implementation; the only guarantee is that {@link
* KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist.
*
* @param columnFamilyHandle the {@link ColumnFamilyHandle} to look for the key in
* @param readOptions the {@link ReadOptions} to use when reading the key/value
* @param key bytebuffer containing the value of the key
* @param value bytebuffer which will receive a value if the key exists and a value is known
* @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided
*/
public KeyMayExist keyMayExist(final ColumnFamilyHandle columnFamilyHandle,
final ReadOptions readOptions, final ByteBuffer key, final ByteBuffer value) {
assert key != null : "key ByteBuffer parameter cannot be null";
assert key.isDirect() : "key parameter must be a direct ByteBuffer";
assert value
!= null
: "value ByteBuffer parameter cannot be null. If you do not need the value, use a different version of the method";
assert value.isDirect() : "value parameter must be a direct ByteBuffer";
final int[] result = keyMayExistDirectFoundValue(nativeHandle_,
columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_,
readOptions == null ? 0 : readOptions.nativeHandle_, key, key.position(), key.remaining(),
value, value.position(), value.remaining());
final int valueLength = result[1];
value.limit(value.position() + Math.min(valueLength, value.remaining()));
return new KeyMayExist(KeyMayExist.KeyMayExistEnum.values()[result[0]], valueLength);
}
/**
* <p>Return a heap-allocated iterator over the contents of the
* database. The result of newIterator() is initially invalid
......@@ -4674,6 +4826,11 @@ public class RocksDB extends RocksObject {
private native int getDirect(long handle, long readOptHandle, ByteBuffer key, int keyOffset,
int keyLength, ByteBuffer value, int valueOffset, int valueLength, long cfHandle)
throws RocksDBException;
private native boolean keyMayExistDirect(final long handle, final long cfHhandle,
final long readOptHandle, final ByteBuffer key, final int keyOffset, final int keyLength);
private native int[] keyMayExistDirectFoundValue(final long handle, final long cfHhandle,
final long readOptHandle, final ByteBuffer key, final int keyOffset, final int keyLength,
final ByteBuffer value, final int valueOffset, final int valueLength);
private native void deleteDirect(long handle, long optHandle, ByteBuffer key, int keyOffset,
int keyLength, long cfHandle) throws RocksDBException;
private native long getLongProperty(final long nativeHandle,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册