提交 6b72342a 编写于 作者: T Tomasz Posluszny 提交者: Facebook GitHub Bot

Implement missing Java API for ColumnFamilyOptions (#7372)

Summary:
Covered methods:
- OldDefaults()
- OptimizeForSmallDb(std::shared_ptr<Cache>)

Covered fields:
- cf_paths

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7372

Reviewed By: pdillinger

Differential Revision: D23683449

Pulled By: jay-zhuang

fbshipit-source-id: 3e5a8b657cc382c19de3a48c666a3b0e8d96968d
上级 ecc8ffe1
......@@ -917,6 +917,134 @@ jstring Java_org_rocksdb_Options_memTableFactoryName(
return env->NewStringUTF(tf->Name());
}
static std::vector<ROCKSDB_NAMESPACE::DbPath>
rocksdb_convert_cf_paths_from_java_helper(JNIEnv* env, jobjectArray path_array,
jlongArray size_array,
jboolean* has_exception) {
jboolean copy_str_has_exception;
std::vector<std::string> paths = ROCKSDB_NAMESPACE::JniUtil::copyStrings(
env, path_array, &copy_str_has_exception);
if (JNI_TRUE == copy_str_has_exception) {
// Exception thrown
*has_exception = JNI_TRUE;
return {};
}
if (static_cast<size_t>(env->GetArrayLength(size_array)) != paths.size()) {
ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(
env,
ROCKSDB_NAMESPACE::Status::InvalidArgument(
ROCKSDB_NAMESPACE::Slice("There should be a corresponding target "
"size for every path and vice versa.")));
*has_exception = JNI_TRUE;
return {};
}
jlong* size_array_ptr = env->GetLongArrayElements(size_array, nullptr);
if (nullptr == size_array_ptr) {
// exception thrown: OutOfMemoryError
return {};
}
std::vector<ROCKSDB_NAMESPACE::DbPath> cf_paths;
for (size_t i = 0; i < paths.size(); ++i) {
jlong target_size = size_array_ptr[i];
if (target_size < 0) {
ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(
env,
ROCKSDB_NAMESPACE::Status::InvalidArgument(ROCKSDB_NAMESPACE::Slice(
"Path target size has to be positive.")));
*has_exception = JNI_TRUE;
env->ReleaseLongArrayElements(size_array, size_array_ptr, JNI_ABORT);
return {};
}
cf_paths.push_back(ROCKSDB_NAMESPACE::DbPath(
paths[i], static_cast<uint64_t>(target_size)));
}
env->ReleaseLongArrayElements(size_array, size_array_ptr, JNI_ABORT);
return cf_paths;
}
/*
* Class: org_rocksdb_Options
* Method: setCfPaths
* Signature: (J[Ljava/lang/String;[J)V
*/
void Java_org_rocksdb_Options_setCfPaths(JNIEnv* env, jclass, jlong jhandle,
jobjectArray path_array,
jlongArray size_array) {
auto* options = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
jboolean has_exception;
std::vector<ROCKSDB_NAMESPACE::DbPath> cf_paths =
rocksdb_convert_cf_paths_from_java_helper(env, path_array, size_array,
&has_exception);
if (JNI_FALSE == has_exception) {
options->cf_paths = std::move(cf_paths);
}
}
/*
* Class: org_rocksdb_Options
* Method: cfPathsLen
* Signature: (J)J
*/
jlong Java_org_rocksdb_Options_cfPathsLen(JNIEnv*, jclass, jlong jhandle) {
auto* opt = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
return static_cast<jlong>(opt->cf_paths.size());
}
template <typename T>
static void rocksdb_convert_cf_paths_to_java_helper(JNIEnv* env, jlong jhandle,
jobjectArray jpaths,
jlongArray jtarget_sizes) {
jboolean is_copy;
jlong* ptr_jtarget_size = env->GetLongArrayElements(jtarget_sizes, &is_copy);
if (ptr_jtarget_size == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
auto* opt = reinterpret_cast<T*>(jhandle);
const jsize len = env->GetArrayLength(jpaths);
for (jsize i = 0; i < len; i++) {
ROCKSDB_NAMESPACE::DbPath cf_path = opt->cf_paths[i];
jstring jpath = env->NewStringUTF(cf_path.path.c_str());
if (jpath == nullptr) {
// exception thrown: OutOfMemoryError
env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT);
return;
}
env->SetObjectArrayElement(jpaths, i, jpath);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
env->DeleteLocalRef(jpath);
env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT);
return;
}
ptr_jtarget_size[i] = static_cast<jint>(cf_path.target_size);
env->DeleteLocalRef(jpath);
}
env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size,
is_copy ? 0 : JNI_ABORT);
}
/*
* Class: org_rocksdb_Options
* Method: cfPaths
* Signature: (J[Ljava/lang/String;[J)V
*/
void Java_org_rocksdb_Options_cfPaths(JNIEnv* env, jclass, jlong jhandle,
jobjectArray jpaths,
jlongArray jtarget_sizes) {
rocksdb_convert_cf_paths_to_java_helper<ROCKSDB_NAMESPACE::Options>(
env, jhandle, jpaths, jtarget_sizes);
}
/*
* Class: org_rocksdb_Options
* Method: setMaxManifestFileSize
......@@ -2860,16 +2988,45 @@ void Java_org_rocksdb_Options_setOptimizeFiltersForHits(
static_cast<bool>(joptimize_filters_for_hits);
}
/*
* Class: org_rocksdb_Options
* Method: oldDefaults
* Signature: (JII)V
*/
void Java_org_rocksdb_Options_oldDefaults(JNIEnv*, jclass, jlong jhandle,
jint major_version,
jint minor_version) {
reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)->OldDefaults(
major_version, minor_version);
}
/*
* Class: org_rocksdb_Options
* Method: optimizeForSmallDb
* Signature: (J)V
*/
void Java_org_rocksdb_Options_optimizeForSmallDb(
JNIEnv*, jobject, jlong jhandle) {
void Java_org_rocksdb_Options_optimizeForSmallDb__J(JNIEnv*, jobject,
jlong jhandle) {
reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)->OptimizeForSmallDb();
}
/*
* Class: org_rocksdb_Options
* Method: optimizeForSmallDb
* Signature: (JJ)V
*/
void Java_org_rocksdb_Options_optimizeForSmallDb__JJ(JNIEnv*, jclass,
jlong jhandle,
jlong cache_handle) {
auto* cache_sptr_ptr =
reinterpret_cast<std::shared_ptr<ROCKSDB_NAMESPACE::Cache>*>(
cache_handle);
auto* options_ptr = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
auto* cf_options_ptr =
static_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(options_ptr);
cf_options_ptr->OptimizeForSmallDb(cache_sptr_ptr);
}
/*
* Class: org_rocksdb_Options
* Method: optimizeForPointLookup
......@@ -3381,17 +3538,45 @@ void Java_org_rocksdb_ColumnFamilyOptions_disposeInternal(
delete cfo;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: oldDefaults
* Signature: (JII)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_oldDefaults(JNIEnv*, jclass,
jlong jhandle,
jint major_version,
jint minor_version) {
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
->OldDefaults(major_version, minor_version);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: optimizeForSmallDb
* Signature: (J)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb(
JNIEnv*, jobject, jlong jhandle) {
void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__J(JNIEnv*,
jobject,
jlong jhandle) {
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
->OptimizeForSmallDb();
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: optimizeForSmallDb
* Signature: (JJ)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__JJ(
JNIEnv*, jclass, jlong jhandle, jlong cache_handle) {
auto* cache_sptr_ptr =
reinterpret_cast<std::shared_ptr<ROCKSDB_NAMESPACE::Cache>*>(
cache_handle);
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
->OptimizeForSmallDb(cache_sptr_ptr);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: optimizeForPointLookup
......@@ -3695,6 +3880,52 @@ jstring Java_org_rocksdb_ColumnFamilyOptions_tableFactoryName(
return env->NewStringUTF(tf->Name());
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setCfPaths
* Signature: (J[Ljava/lang/String;[J)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setCfPaths(JNIEnv* env, jclass,
jlong jhandle,
jobjectArray path_array,
jlongArray size_array) {
auto* options =
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle);
jboolean has_exception;
std::vector<ROCKSDB_NAMESPACE::DbPath> cf_paths =
rocksdb_convert_cf_paths_from_java_helper(env, path_array, size_array,
&has_exception);
if (JNI_FALSE == has_exception) {
options->cf_paths = std::move(cf_paths);
}
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: cfPathsLen
* Signature: (J)J
*/
jlong Java_org_rocksdb_ColumnFamilyOptions_cfPathsLen(JNIEnv*, jclass,
jlong jhandle) {
auto* opt =
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle);
return static_cast<jlong>(opt->cf_paths.size());
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: cfPaths
* Signature: (J[Ljava/lang/String;[J)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_cfPaths(JNIEnv* env, jclass,
jlong jhandle,
jobjectArray jpaths,
jlongArray jtarget_sizes) {
rocksdb_convert_cf_paths_to_java_helper<
ROCKSDB_NAMESPACE::ColumnFamilyOptions>(env, jhandle, jpaths,
jtarget_sizes);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: minWriteBufferNumberToMerge
......
......@@ -5,9 +5,8 @@
package org.rocksdb;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.nio.file.Paths;
import java.util.*;
/**
* ColumnFamilyOptions to control the behavior of a database. It will be used
......@@ -137,12 +136,24 @@ public class ColumnFamilyOptions extends RocksObject
return columnFamilyOptions;
}
@Override
public ColumnFamilyOptions oldDefaults(final int majorVersion, final int minorVersion) {
oldDefaults(nativeHandle_, majorVersion, minorVersion);
return this;
}
@Override
public ColumnFamilyOptions optimizeForSmallDb() {
optimizeForSmallDb(nativeHandle_);
return this;
}
@Override
public ColumnFamilyOptions optimizeForSmallDb(final Cache cache) {
optimizeForSmallDb(nativeHandle_, cache.getNativeHandle());
return this;
}
@Override
public ColumnFamilyOptions optimizeForPointLookup(
final long blockCacheSizeMb) {
......@@ -596,6 +607,45 @@ public class ColumnFamilyOptions extends RocksObject
return tableFactoryName(nativeHandle_);
}
@Override
public ColumnFamilyOptions setCfPaths(final Collection<DbPath> cfPaths) {
assert (isOwningHandle());
final int len = cfPaths.size();
final String paths[] = new String[len];
final long targetSizes[] = new long[len];
int i = 0;
for (final DbPath dbPath : cfPaths) {
paths[i] = dbPath.path.toString();
targetSizes[i] = dbPath.targetSize;
i++;
}
setCfPaths(nativeHandle_, paths, targetSizes);
return this;
}
@Override
public List<DbPath> cfPaths() {
final int len = (int) cfPathsLen(nativeHandle_);
if (len == 0) {
return Collections.emptyList();
}
final String paths[] = new String[len];
final long targetSizes[] = new long[len];
cfPaths(nativeHandle_, paths, targetSizes);
final List<DbPath> cfPaths = new ArrayList<>();
for (int i = 0; i < len; i++) {
cfPaths.add(new DbPath(Paths.get(paths[i]), targetSizes[i]));
}
return cfPaths;
}
@Override
public ColumnFamilyOptions setInplaceUpdateSupport(
final boolean inplaceUpdateSupport) {
......@@ -881,7 +931,10 @@ public class ColumnFamilyOptions extends RocksObject
final long optionsHandle);
@Override protected final native void disposeInternal(final long handle);
private static native void oldDefaults(
final long handle, final int majorVersion, final int minorVersion);
private native void optimizeForSmallDb(final long handle);
private static native void optimizeForSmallDb(final long handle, final long cacheHandle);
private native void optimizeForPointLookup(long handle,
long blockCacheSizeMb);
private native void optimizeLevelStyleCompaction(long handle,
......@@ -970,6 +1023,11 @@ public class ColumnFamilyOptions extends RocksObject
private native String memTableFactoryName(long handle);
private native void setTableFactory(long handle, long factoryHandle);
private native String tableFactoryName(long handle);
private static native void setCfPaths(
final long handle, final String[] paths, final long[] targetSizes);
private static native long cfPathsLen(final long handle);
private static native void cfPaths(
final long handle, final String[] paths, final long[] targetSizes);
private native void setInplaceUpdateSupport(
long handle, boolean inplaceUpdateSupport);
private native boolean inplaceUpdateSupport(long handle);
......
......@@ -5,8 +5,19 @@
package org.rocksdb;
import java.util.Collection;
import java.util.List;
public interface ColumnFamilyOptionsInterface<T extends ColumnFamilyOptionsInterface<T>>
extends AdvancedColumnFamilyOptionsInterface<T> {
/**
* The function recovers options to a previous version. Only 4.6 or later
* versions are supported.
*
* @return the instance of the current object.
*/
T oldDefaults(int majorVersion, int minorVersion);
/**
* Use this if your DB is very small (like under 1GB) and you don't want to
* spend lots of memory for memtables.
......@@ -15,6 +26,16 @@ public interface ColumnFamilyOptionsInterface<T extends ColumnFamilyOptionsInter
*/
T optimizeForSmallDb();
/**
* Some functions that make it easier to optimize RocksDB
* Use this if your DB is very small (like under 1GB) and you don't want to
* spend lots of memory for memtables.
* An optional cache object is passed in to be used as the block cache
*
* @return the instance of the current object.
*/
T optimizeForSmallDb(Cache cache);
/**
* Use this if you don't need to keep the data sorted, i.e. you'll never use
* an iterator, only Put() and Get() API calls
......@@ -371,6 +392,30 @@ public interface ColumnFamilyOptionsInterface<T extends ColumnFamilyOptionsInter
*/
String tableFactoryName();
/**
* A list of paths where SST files for this column family
* can be put into, with its target size. Similar to db_paths,
* newer data is placed into paths specified earlier in the
* vector while older data gradually moves to paths specified
* later in the vector.
* Note that, if a path is supplied to multiple column
* families, it would have files and total size from all
* the column families combined. User should provision for the
* total size(from all the column families) in such cases.
*
* If left empty, db_paths will be used.
* Default: empty
*
* @param paths collection of paths for SST files.
* @return the reference of the current options.
*/
T setCfPaths(final Collection<DbPath> paths);
/**
* @return collection of paths for SST files.
*/
List<DbPath> cfPaths();
/**
* Compression algorithm that will be used for the bottommost level that
* contain files. If level-compaction is used, this option will only affect
......
......@@ -157,12 +157,24 @@ public class Options extends RocksObject
return createMissingColumnFamilies(nativeHandle_);
}
@Override
public Options oldDefaults(final int majorVersion, final int minorVersion) {
oldDefaults(nativeHandle_, majorVersion, minorVersion);
return this;
}
@Override
public Options optimizeForSmallDb() {
optimizeForSmallDb(nativeHandle_);
return this;
}
@Override
public Options optimizeForSmallDb(final Cache cache) {
optimizeForSmallDb(nativeHandle_, cache.getNativeHandle());
return this;
}
@Override
public Options optimizeForPointLookup(
long blockCacheSizeMb) {
......@@ -1284,6 +1296,45 @@ public class Options extends RocksObject
return tableFactoryName(nativeHandle_);
}
@Override
public Options setCfPaths(final Collection<DbPath> cfPaths) {
assert (isOwningHandle());
final int len = cfPaths.size();
final String paths[] = new String[len];
final long targetSizes[] = new long[len];
int i = 0;
for (final DbPath dbPath : cfPaths) {
paths[i] = dbPath.path.toString();
targetSizes[i] = dbPath.targetSize;
i++;
}
setCfPaths(nativeHandle_, paths, targetSizes);
return this;
}
@Override
public List<DbPath> cfPaths() {
final int len = (int) cfPathsLen(nativeHandle_);
if (len == 0) {
return Collections.emptyList();
}
final String paths[] = new String[len];
final long targetSizes[] = new long[len];
cfPaths(nativeHandle_, paths, targetSizes);
final List<DbPath> cfPaths = new ArrayList<>();
for (int i = 0; i < len; i++) {
cfPaths.add(new DbPath(Paths.get(paths[i]), targetSizes[i]));
}
return cfPaths;
}
@Override
public Options useFixedLengthPrefixExtractor(final int n) {
assert(isOwningHandle());
......@@ -2052,7 +2103,10 @@ public class Options extends RocksObject
// CF native handles
private static native void oldDefaults(
final long handle, final int majorVersion, final int minorVersion);
private native void optimizeForSmallDb(final long handle);
private static native void optimizeForSmallDb(final long handle, final long cacheHandle);
private native void optimizeForPointLookup(long handle,
long blockCacheSizeMb);
private native void optimizeLevelStyleCompaction(long handle,
......@@ -2139,6 +2193,11 @@ public class Options extends RocksObject
private native String memTableFactoryName(long handle);
private native void setTableFactory(long handle, long factoryHandle);
private native String tableFactoryName(long handle);
private static native void setCfPaths(
final long handle, final String[] paths, final long[] targetSizes);
private static native long cfPathsLen(final long handle);
private static native void cfPaths(
final long handle, final String[] paths, final long[] targetSizes);
private native void setInplaceUpdateSupport(
long handle, boolean inplaceUpdateSupport);
private native boolean inplaceUpdateSupport(long handle);
......
......@@ -38,4 +38,8 @@ public abstract class RocksObject extends AbstractImmutableNativeReference {
}
protected abstract void disposeInternal(final long handle);
public long getNativeHandle() {
return nativeHandle_;
}
}
......@@ -5,17 +5,17 @@
package org.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import org.junit.ClassRule;
import org.junit.Test;
import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
public class ColumnFamilyOptionsTest {
@ClassRule
......@@ -652,4 +652,37 @@ public class ColumnFamilyOptionsTest {
assertThat(options.compactionThreadLimiter()).isEqualTo(compactionThreadLimiter);
}
}
@Test
public void oldDefaults() {
try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) {
options.oldDefaults(4, 6);
assertEquals(4 << 20, options.writeBufferSize());
assertThat(options.compactionPriority()).isEqualTo(CompactionPriority.ByCompensatedSize);
assertThat(options.targetFileSizeBase()).isEqualTo(2 * 1048576);
assertThat(options.maxBytesForLevelBase()).isEqualTo(10 * 1048576);
assertThat(options.softPendingCompactionBytesLimit()).isEqualTo(0);
assertThat(options.hardPendingCompactionBytesLimit()).isEqualTo(0);
assertThat(options.level0StopWritesTrigger()).isEqualTo(24);
}
}
@Test
public void optimizeForSmallDbWithCache() {
try (final ColumnFamilyOptions options = new ColumnFamilyOptions();
final Cache cache = new LRUCache(1024)) {
assertThat(options.optimizeForSmallDb(cache)).isEqualTo(options);
}
}
@Test
public void cfPaths() throws IOException {
try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) {
final List<DbPath> paths = Arrays.asList(
new DbPath(Paths.get("test1"), 2 << 25), new DbPath(Paths.get("/test2/path"), 2 << 25));
assertThat(options.cfPaths()).isEqualTo(Collections.emptyList());
assertThat(options.setCfPaths(paths)).isEqualTo(options);
assertThat(options.cfPaths()).isEqualTo(paths);
}
}
}
......@@ -5,16 +5,18 @@
package org.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import org.junit.ClassRule;
import org.junit.Test;
import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory;
import static org.assertj.core.api.Assertions.assertThat;
public class OptionsTest {
@ClassRule
......@@ -1317,4 +1319,36 @@ public class OptionsTest {
assertThat(options.compactionThreadLimiter()).isEqualTo(compactionThreadLimiter);
}
}
@Test
public void oldDefaults() {
try (final Options options = new Options()) {
options.oldDefaults(4, 6);
assertThat(options.writeBufferSize()).isEqualTo(4 << 20);
assertThat(options.compactionPriority()).isEqualTo(CompactionPriority.ByCompensatedSize);
assertThat(options.targetFileSizeBase()).isEqualTo(2 * 1048576);
assertThat(options.maxBytesForLevelBase()).isEqualTo(10 * 1048576);
assertThat(options.softPendingCompactionBytesLimit()).isEqualTo(0);
assertThat(options.hardPendingCompactionBytesLimit()).isEqualTo(0);
assertThat(options.level0StopWritesTrigger()).isEqualTo(24);
}
}
@Test
public void optimizeForSmallDbWithCache() {
try (final Options options = new Options(); final Cache cache = new LRUCache(1024)) {
assertThat(options.optimizeForSmallDb(cache)).isEqualTo(options);
}
}
@Test
public void cfPaths() throws IOException {
try (final Options options = new Options()) {
final List<DbPath> paths = Arrays.asList(
new DbPath(Paths.get("test1"), 2 << 25), new DbPath(Paths.get("/test2/path"), 2 << 25));
assertThat(options.cfPaths()).isEqualTo(Collections.emptyList());
assertThat(options.setCfPaths(paths)).isEqualTo(options);
assertThat(options.cfPaths()).isEqualTo(paths);
}
}
}
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册