diff --git a/core/src/main/java/com/questdb/PropServerConfiguration.java b/core/src/main/java/com/questdb/PropServerConfiguration.java new file mode 100644 index 0000000000000000000000000000000000000000..c93ff3b79a6294def1ddc010426b190852953757 --- /dev/null +++ b/core/src/main/java/com/questdb/PropServerConfiguration.java @@ -0,0 +1,826 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (C) 2014-2019 Appsicle + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + ******************************************************************************/ + +package com.questdb; + +import com.questdb.cairo.CairoConfiguration; +import com.questdb.cutlass.http.HttpServerConfiguration; +import com.questdb.cutlass.http.MimeTypesCache; +import com.questdb.cutlass.http.processors.StaticContentProcessorConfiguration; +import com.questdb.cutlass.http.processors.TextImportProcessorConfiguration; +import com.questdb.cutlass.line.udp.LineUdpReceiverConfiguration; +import com.questdb.cutlass.text.TextConfiguration; +import com.questdb.network.*; +import com.questdb.std.*; +import com.questdb.std.microtime.MicrosecondClock; +import com.questdb.std.microtime.MicrosecondClockImpl; +import com.questdb.std.str.Path; +import com.questdb.std.time.MillisecondClock; +import com.questdb.std.time.MillisecondClockImpl; + +import java.util.Properties; + +public class PropServerConfiguration implements ServerConfigurationV2 { + private final IODispatcherConfiguration ioDispatcherConfiguration = new PropIODispatcherConfiguration(); + private final TextImportProcessorConfiguration textImportProcessorConfiguration = new PropTextImportProcessorConfiguration(); + private final StaticContentProcessorConfiguration staticContentProcessorConfiguration = new PropStaticContentProcessorConfiguration(); + private final HttpServerConfiguration httpServerConfiguration = new PropHttpServerConfiguration(); + private final TextConfiguration textConfiguration = new PropTextConfiguration(); + private final CairoConfiguration cairoConfiguration = new PropCairoConfiguration(); + private final LineUdpReceiverConfiguration lineUdpReceiverConfiguration = new PropLineUdpReceiverConfiguration(); + private final int connectionPoolInitialCapacity; + private final int connectionStringPoolCapacity; + private final int multipartHeaderBufferSize; + private final long multipartIdleSpinCount; + private final int recvBufferSize; + private final int requestHeaderBufferSize; + private final int responseHeaderBufferSize; + private final int workerCount; + private final int sendBufferSize; + private final CharSequence indexFileName; + private final CharSequence publicDirectory; + private final boolean abortBrokenUploads; + private final int activeConnectionLimit; + private final int eventCapacity; + private final int ioQueueCapacity; + private final long idleConnectionTimeout; + private final int interestQueueCapacity; + private final int listenBacklog; + private final int sndBufSize; + private final int rcvBufSize; + private final String adapterSetConfigurationFileName; + private final int dateAdapterPoolCapacity; + private final int jsonCacheLimit; + private final int jsonCacheSize; + private final double maxRequiredDelimiterStdDev; + private final int metadataStringPoolCapacity; + private final int rollBufferLimit; + private final int rollBufferSize; + private final int textAnalysisMaxLines; + private final int textLexerStringPoolCapacity; + private final int timestampAdapterPoolCapacity; + private final int utf8SinkSize; + private final int createAsSelectRetryCount; + private final CharSequence defaultMapType; + private final boolean defaultSymbolCacheFlag; + private final int defaultSymbolCapacity; + private final int fileOperationRetryCount; + private final long idleCheckInterval; + private final long inactiveReaderTTL; + private final long inactiveWriterTTL; + private final int indexValueBlockSize; + private final int maxSwapFileCount; + private final int mkdirMode; + private final int parallelIndexThreshold; + private final int readerPoolMaxSegments; + private final CharSequence root; + private final long spinLockTimeoutUs; + private final int sqlCacheRows; + private final int sqlCacheBlocks; + private final int sqlCharacterStoreCapacity; + private final int sqlCharacterStoreSequencePoolCapacity; + private final int sqlColumnPoolCapacity; + private final double sqlCompactMapLoadFactor; + private final int sqlExpressionPoolCapacity; + private final double sqlFastMapLoadFactor; + private final int sqlJoinContextPoolCapacity; + private final int sqlLexerPoolCapacity; + private final int sqlMapKeyCapacity; + private final int sqlMapPageSize; + private final int sqlModelPoolCapacity; + private final int sqlSortKeyPageSize; + private final int sqlSortLightValuePageSize; + private final int sqlHashJoinValuePageSize; + private final int sqlTreePageSize; + private final int sqlHashJoinLightValuePageSize; + private final int sqlSortValuePageSize; + private final long workStealTimeoutNanos; + private final boolean parallelIndexingEnabled; + private final int sqlJoinMetadataPageSize; + private final int lineUdpCommitRate; + private final int lineUdpGroupIPv4Address; + private final int lineUdpMsgBufferSize; + private final int lineUdpMsgCount; + private final int lineUdpReceiveBufferSize; + private final MimeTypesCache mimeTypesCache; + private int bindIPv4Address; + private int bindPort; + private int lineUdpBindIPV4Address; + private int lineUdpPort; + + public PropServerConfiguration(String root, Properties properties) throws ServerConfigurationException { + this.root = root; + this.connectionPoolInitialCapacity = getInt(properties, "http.connection.pool.initial.capacity", 16); + this.connectionStringPoolCapacity = getInt(properties, "http.connection.string.pool.capacity", 128); + this.multipartHeaderBufferSize = getIntSize(properties, "http.multipart.header.buffer.size", 512); + this.multipartIdleSpinCount = getLong(properties, "http.multipart.idle.spin.count", 10_000); + this.recvBufferSize = getIntSize(properties, "http.receive.buffer.size", 1024 * 1024); + this.requestHeaderBufferSize = getIntSize(properties, "http.request.header.buffer.size", 1024); + this.responseHeaderBufferSize = getIntSize(properties, "http.response.header.buffer.size", 1024 * 1024); + this.workerCount = getInt(properties, "http.worker.count", 2); + this.sendBufferSize = getIntSize(properties, "http.send.buffer.size", 2 * 1024 * 1024); + this.indexFileName = getString(properties, "http.static.index.file.name", "index.html"); + this.publicDirectory = getString(properties, "http.static.pubic.directory", "public"); + this.abortBrokenUploads = getBoolean(properties, "http.text.abort.broken.uploads", true); + this.activeConnectionLimit = getInt(properties, "http.net.active.connection.limit", 256); + this.eventCapacity = getInt(properties, "http.net.event.capacity", 1024); + this.ioQueueCapacity = getInt(properties, "http.net.io.queue.capacity", 1024); + this.idleConnectionTimeout = getLong(properties, "http.net.idle.connection.timeout", 5 * 60 * 1000L); + this.interestQueueCapacity = getInt(properties, "http.net.interest.queue.capacity", 1024); + this.listenBacklog = getInt(properties, "http.net.listen.backlog", 256); + this.sndBufSize = getIntSize(properties, "http.net.snd.buf.size", 2 * 1024 * 1024); + this.rcvBufSize = getIntSize(properties, "http.net.rcv.buf.size", 2 * 1024 * 1024); + this.adapterSetConfigurationFileName = getString(properties, "http.text.adapter.set.config", "/text_loader.json"); + this.dateAdapterPoolCapacity = getInt(properties, "http.text.date.adapter.pool.capacity", 16); + this.jsonCacheLimit = getIntSize(properties, "http.text.json.cache.limit", 16384); + this.jsonCacheSize = getIntSize(properties, "http.text.json.cache.size", 8192); + this.maxRequiredDelimiterStdDev = getDouble(properties, "http.text.max.required.delimiter.stddev", 0.1222d); + this.metadataStringPoolCapacity = getInt(properties, "http.text.metadata.string.pool.capacity", 128); + + this.rollBufferLimit = getIntSize(properties, "http.text.roll.buffer.limit", 4096); + this.rollBufferSize = getIntSize(properties, "http.text.roll.buffer.size", 1024); + this.textAnalysisMaxLines = getInt(properties, "http.text.analysis.max.lines", 1000); + this.textLexerStringPoolCapacity = getInt(properties, "http.text.lexer.string.pool.capacity", 64); + this.timestampAdapterPoolCapacity = getInt(properties, "http.text.timestamp.adapter.pool.capacity", 64); + this.utf8SinkSize = getIntSize(properties, "http.text.utf8.sink.size", 4096); + + parseBindTo(properties, "http.bind.to", "0.0.0.0:9000", (a, p) -> { + bindIPv4Address = a; + bindPort = p; + }); + + String defaultFilePath = this.getClass().getResource("/site/conf/mime.types").getFile(); + if (Os.type == Os.WINDOWS) { + // on Windows Java returns "/C:/dir/file". This leading slash is Java specific and doesn't bode well + // with OS file open methods. + defaultFilePath = defaultFilePath.substring(1); + } + try (Path path = new Path().of(defaultFilePath).$()) { + this.mimeTypesCache = new MimeTypesCache(FilesFacadeImpl.INSTANCE, path); + } + + this.createAsSelectRetryCount = getInt(properties, "cairo.create.as.select.retry.count", 5); + this.defaultMapType = getString(properties, "cairo.default.map.type", "fast"); + this.defaultSymbolCacheFlag = getBoolean(properties, "cairo.default.symbol.cache.flag", false); + this.defaultSymbolCapacity = getInt(properties, "cairo.default.symbol.capacity", 256); + this.fileOperationRetryCount = getInt(properties, "cairo.file.operation.retry.count", 30); + this.idleCheckInterval = getLong(properties, "cairo.idle.check.interval", 100); + this.inactiveReaderTTL = getLong(properties, "cairo.inactive.reader.ttl", -10000); + this.inactiveWriterTTL = getLong(properties, "cairo.inactive.writer.ttl", -10000); + this.indexValueBlockSize = Numbers.ceilPow2(getIntSize(properties, "cairo.index.value.block.size", 256)); + this.maxSwapFileCount = getInt(properties, "cairo.max.swap.file.count", 30); + this.mkdirMode = getInt(properties, "cairo.mkdir.mode", 509); + this.parallelIndexThreshold = getInt(properties, "cairo.parallel.index.threshold", 100000); + this.readerPoolMaxSegments = getInt(properties, "cairo.reader.pool.max.segments", 5); + this.spinLockTimeoutUs = getLong(properties, "cairo.spin.lock.timeout", 1_000_000); + this.sqlCacheRows = getInt(properties, "cairo.cache.rows", 16); + this.sqlCacheBlocks = getIntSize(properties, "cairo.cache.blocks", 4); + this.sqlCharacterStoreCapacity = getInt(properties, "cairo.character.store.capacity", 1024); + this.sqlCharacterStoreSequencePoolCapacity = getInt(properties, "cairo.character.store.sequence.pool.capacity", 64); + this.sqlColumnPoolCapacity = getInt(properties, "cairo.column.pool.capacity", 4096); + this.sqlCompactMapLoadFactor = getDouble(properties, "cairo.compact.map.load.factor", 0.7); + this.sqlExpressionPoolCapacity = getInt(properties, "cairo.expression.pool.capacity", 8192); + this.sqlFastMapLoadFactor = getDouble(properties, "cairo.fast.map.load.factor", 0.5); + this.sqlJoinContextPoolCapacity = getInt(properties, "cairo.sql.join.context.pool.capacity", 64); + this.sqlLexerPoolCapacity = getInt(properties, "cairo.lexer.pool.capacity", 2048); + this.sqlMapKeyCapacity = getInt(properties, "cairo.sql.map.key.capacity", 2048); + this.sqlMapPageSize = getIntSize(properties, "cairo.sql.map.page.size", 4 * 1024 * 1024); + this.sqlModelPoolCapacity = getInt(properties, "cairo.model.pool.capacity", 1024); + this.sqlSortKeyPageSize = getIntSize(properties, "cairo.sql.sort.key.page.size", 4 * 1024 * 1024); + this.sqlSortLightValuePageSize = getIntSize(properties, "cairo.sql.sort.light.value.page.size", 1048576); + this.sqlHashJoinValuePageSize = getIntSize(properties, "cairo.sql.hash.join.value.page.size", 16777216); + this.sqlTreePageSize = getIntSize(properties, "cairo.sql.tree.page.size", 4 * 1024 * 1024); + this.sqlHashJoinLightValuePageSize = getIntSize(properties, "cairo.sql.hash.join.light.value.page.size", 1048576); + this.sqlSortValuePageSize = getIntSize(properties, "cairo.sql.sort.value.page.size", 16777216); + this.workStealTimeoutNanos = getLong(properties, "cairo.work.steal.timeout.nanos", 10_000); + this.parallelIndexingEnabled = getBoolean(properties, "cairo.parallel.indexing.enabled", true); + this.sqlJoinMetadataPageSize = getIntSize(properties, "cairo.sql.join.metadata.page.size", 16384); + + parseBindTo(properties, "line.udp.bind.to", "0.0.0.0:9009", (a, p) -> { + this.lineUdpBindIPV4Address = a; + this.lineUdpPort = p; + }); + + this.lineUdpGroupIPv4Address = getIPv4Address(properties, "line.udp.join", "232.1.2.3"); + this.lineUdpCommitRate = getInt(properties, "line.udp.commit.rate", 10_000); + this.lineUdpMsgBufferSize = getIntSize(properties, "line.udp.msg.buffer.size", 1024 * 1024); + this.lineUdpMsgCount = getInt(properties, "line.udp.msg.count", 10_000); + this.lineUdpReceiveBufferSize = getIntSize(properties, "line.udp.receive.buffer.size", 2048); + } + + @Override + public CairoConfiguration getCairoConfiguration() { + return cairoConfiguration; + } + + @Override + public HttpServerConfiguration getHttpServerConfiguration() { + return httpServerConfiguration; + } + + @Override + public LineUdpReceiverConfiguration getLineUdpReceiverConfiguration() { + return lineUdpReceiverConfiguration; + } + + private boolean getBoolean(Properties properties, String key, boolean defaultValue) { + final String value = properties.getProperty(key); + return value == null ? defaultValue : Boolean.parseBoolean(value); + } + + private double getDouble(Properties properties, String key, double defaultValue) throws ServerConfigurationException { + final String value = properties.getProperty(key); + try { + return value != null ? Numbers.parseDouble(value) : defaultValue; + } catch (NumericException e) { + throw new ServerConfigurationException(key, value); + } + } + + @SuppressWarnings("SameParameterValue") + private int getIPv4Address(Properties properties, String key, String defaultValue) throws ServerConfigurationException { + final String value = getString(properties, key, defaultValue); + try { + return Net.parseIPv4(value); + } catch (NetworkError e) { + throw new ServerConfigurationException(key, value); + } + } + + private int getInt(Properties properties, String key, int defaultValue) throws ServerConfigurationException { + final String value = properties.getProperty(key); + try { + return value != null ? Numbers.parseInt(value) : defaultValue; + } catch (NumericException e) { + throw new ServerConfigurationException(key, value); + } + } + + private int getIntSize(Properties properties, String key, int defaultValue) throws ServerConfigurationException { + final String value = properties.getProperty(key); + try { + return value != null ? Numbers.parseIntSize(value) : defaultValue; + } catch (NumericException e) { + throw new ServerConfigurationException(key, value); + } + } + + private long getLong(Properties properties, String key, long defaultValue) throws ServerConfigurationException { + final String value = properties.getProperty(key); + try { + return value != null ? Numbers.parseLong(value) : defaultValue; + } catch (NumericException e) { + throw new ServerConfigurationException(key, value); + } + } + + private String getString(Properties properties, String key, String defaultValue) { + String value = properties.getProperty(key); + if (value == null) { + return defaultValue; + } + return value; + } + + private void parseBindTo( + Properties properties, + String key, + String defaultValue, + BindToParser parser + ) throws ServerConfigurationException { + + final String bindTo = getString(properties, key, defaultValue); + final int colonIndex = bindTo.indexOf(':'); + if (colonIndex == -1) { + throw new ServerConfigurationException(key, bindTo); + } + + final String ipv4Str = bindTo.substring(0, colonIndex); + final int ipv4; + try { + ipv4 = Net.parseIPv4(ipv4Str); + } catch (NetworkError e) { + throw new ServerConfigurationException(key, ipv4Str); + } + + + final String portStr = bindTo.substring(colonIndex + 1); + final int port; + try { + port = Numbers.parseInt(portStr); + } catch (NumericException e) { + throw new ServerConfigurationException(key, portStr); + } + + parser.onReady(ipv4, port); + } + + @FunctionalInterface + private interface BindToParser { + void onReady(int address, int port); + } + + private class PropStaticContentProcessorConfiguration implements StaticContentProcessorConfiguration { + @Override + public FilesFacade getFilesFacade() { + return FilesFacadeImpl.INSTANCE; + } + + @Override + public CharSequence getIndexFileName() { + return indexFileName; + } + + @Override + public MimeTypesCache getMimeTypesCache() { + return mimeTypesCache; + } + + @Override + public CharSequence getPublicDirectory() { + return publicDirectory; + } + } + + private class PropTextImportProcessorConfiguration implements TextImportProcessorConfiguration { + @Override + public boolean abortBrokenUploads() { + return abortBrokenUploads; + } + + @Override + public TextConfiguration getTextConfiguration() { + return textConfiguration; + } + } + + private class PropIODispatcherConfiguration implements IODispatcherConfiguration { + @Override + public int getActiveConnectionLimit() { + return activeConnectionLimit; + } + + @Override + public int getBindIPv4Address() { + return bindIPv4Address; + } + + @Override + public int getBindPort() { + return bindPort; + } + + @Override + public MillisecondClock getClock() { + return MillisecondClockImpl.INSTANCE; + } + + @Override + public int getEventCapacity() { + return eventCapacity; + } + + @Override + public int getIOQueueCapacity() { + return ioQueueCapacity; + } + + @Override + public long getIdleConnectionTimeout() { + return idleConnectionTimeout; + } + + @Override + public int getInterestQueueCapacity() { + return interestQueueCapacity; + } + + @Override + public int getListenBacklog() { + return listenBacklog; + } + + @Override + public NetworkFacade getNetworkFacade() { + return NetworkFacadeImpl.INSTANCE; + } + + @Override + public EpollFacade getEpollFacade() { + return EpollFacadeImpl.INSTANCE; + } + + @Override + public SelectFacade getSelectFacade() { + return SelectFacadeImpl.INSTANCE; + } + + @Override + public int getInitialBias() { + return IOOperation.READ; + } + + @Override + public int getSndBufSize() { + return sndBufSize; + } + + @Override + public int getRcvBufSize() { + return rcvBufSize; + } + } + + private class PropTextConfiguration implements TextConfiguration { + @Override + public String getAdapterSetConfigurationFileName() { + return adapterSetConfigurationFileName; + } + + @Override + public int getDateAdapterPoolCapacity() { + return dateAdapterPoolCapacity; + } + + @Override + public int getJsonCacheLimit() { + return jsonCacheLimit; + } + + @Override + public int getJsonCacheSize() { + return jsonCacheSize; + } + + @Override + public double getMaxRequiredDelimiterStdDev() { + return maxRequiredDelimiterStdDev; + } + + @Override + public int getMetadataStringPoolCapacity() { + return metadataStringPoolCapacity; + } + + @Override + public int getRollBufferLimit() { + return rollBufferLimit; + } + + @Override + public int getRollBufferSize() { + return rollBufferSize; + } + + @Override + public int getTextAnalysisMaxLines() { + return textAnalysisMaxLines; + } + + @Override + public int getTextLexerStringPoolCapacity() { + return textLexerStringPoolCapacity; + } + + @Override + public int getTimestampAdapterPoolCapacity() { + return timestampAdapterPoolCapacity; + } + + @Override + public int getUtf8SinkSize() { + return utf8SinkSize; + } + } + + private class PropHttpServerConfiguration implements HttpServerConfiguration { + @Override + public int getConnectionPoolInitialCapacity() { + return connectionPoolInitialCapacity; + } + + @Override + public int getConnectionStringPoolCapacity() { + return connectionStringPoolCapacity; + } + + @Override + public int getMultipartHeaderBufferSize() { + return multipartHeaderBufferSize; + } + + @Override + public long getMultipartIdleSpinCount() { + return multipartIdleSpinCount; + } + + @Override + public int getRecvBufferSize() { + return recvBufferSize; + } + + @Override + public int getRequestHeaderBufferSize() { + return requestHeaderBufferSize; + } + + @Override + public int getResponseHeaderBufferSize() { + return responseHeaderBufferSize; + } + + @Override + public MillisecondClock getClock() { + return MillisecondClockImpl.INSTANCE; + } + + @Override + public IODispatcherConfiguration getDispatcherConfiguration() { + return ioDispatcherConfiguration; + } + + @Override + public StaticContentProcessorConfiguration getStaticContentProcessorConfiguration() { + return staticContentProcessorConfiguration; + } + + @Override + public TextImportProcessorConfiguration getTextImportProcessorConfiguration() { + return textImportProcessorConfiguration; + } + + @Override + public int getWorkerCount() { + return workerCount; + } + + @Override + public int getSendBufferSize() { + return sendBufferSize; + } + } + + private class PropCairoConfiguration implements CairoConfiguration { + @Override + public int getCreateAsSelectRetryCount() { + return createAsSelectRetryCount; + } + + @Override + public CharSequence getDefaultMapType() { + return defaultMapType; + } + + @Override + public boolean getDefaultSymbolCacheFlag() { + return defaultSymbolCacheFlag; + } + + @Override + public int getDefaultSymbolCapacity() { + return defaultSymbolCapacity; + } + + @Override + public int getFileOperationRetryCount() { + return fileOperationRetryCount; + } + + @Override + public FilesFacade getFilesFacade() { + return FilesFacadeImpl.INSTANCE; + } + + @Override + public long getIdleCheckInterval() { + return idleCheckInterval; + } + + @Override + public long getInactiveReaderTTL() { + return inactiveReaderTTL; + } + + @Override + public long getInactiveWriterTTL() { + return inactiveWriterTTL; + } + + @Override + public int getIndexValueBlockSize() { + return indexValueBlockSize; + } + + @Override + public int getMaxSwapFileCount() { + return maxSwapFileCount; + } + + @Override + public MicrosecondClock getMicrosecondClock() { + return MicrosecondClockImpl.INSTANCE; + } + + @Override + public MillisecondClock getMillisecondClock() { + return MillisecondClockImpl.INSTANCE; + } + + @Override + public int getMkDirMode() { + return mkdirMode; + } + + @Override + public int getParallelIndexThreshold() { + return parallelIndexThreshold; + } + + @Override + public int getReaderPoolMaxSegments() { + return readerPoolMaxSegments; + } + + @Override + public CharSequence getRoot() { + return root; + } + + @Override + public long getSpinLockTimeoutUs() { + return spinLockTimeoutUs; + } + + @Override + public int getSqlCacheBlocks() { + return sqlCacheBlocks; + } + + @Override + public int getSqlCacheRows() { + return sqlCacheRows; + } + + @Override + public int getSqlCharacterStoreCapacity() { + return sqlCharacterStoreCapacity; + } + + @Override + public int getSqlCharacterStoreSequencePoolCapacity() { + return sqlCharacterStoreSequencePoolCapacity; + } + + @Override + public int getSqlColumnPoolCapacity() { + return sqlColumnPoolCapacity; + } + + @Override + public double getSqlCompactMapLoadFactor() { + return sqlCompactMapLoadFactor; + } + + @Override + public int getSqlExpressionPoolCapacity() { + return sqlExpressionPoolCapacity; + } + + @Override + public double getSqlFastMapLoadFactor() { + return sqlFastMapLoadFactor; + } + + @Override + public int getSqlJoinContextPoolCapacity() { + return sqlJoinContextPoolCapacity; + } + + @Override + public int getSqlLexerPoolCapacity() { + return sqlLexerPoolCapacity; + } + + @Override + public int getSqlMapKeyCapacity() { + return sqlMapKeyCapacity; + } + + @Override + public int getSqlMapPageSize() { + return sqlMapPageSize; + } + + @Override + public int getSqlModelPoolCapacity() { + return sqlModelPoolCapacity; + } + + @Override + public int getSqlSortKeyPageSize() { + return sqlSortKeyPageSize; + } + + @Override + public int getSqlSortLightValuePageSize() { + return sqlSortLightValuePageSize; + } + + @Override + public int getSqlHashJoinValuePageSize() { + return sqlHashJoinValuePageSize; + } + + @Override + public int getSqlTreePageSize() { + return sqlTreePageSize; + } + + @Override + public int getSqlHashJoinLightValuePageSize() { + return sqlHashJoinLightValuePageSize; + } + + @Override + public int getSqlSortValuePageSize() { + return sqlSortValuePageSize; + } + + @Override + public long getWorkStealTimeoutNanos() { + return workStealTimeoutNanos; + } + + @Override + public boolean isParallelIndexingEnabled() { + return parallelIndexingEnabled; + } + + @Override + public int getSqlJoinMetadataPageSize() { + return sqlJoinMetadataPageSize; + } + } + + private class PropLineUdpReceiverConfiguration implements LineUdpReceiverConfiguration { + @Override + public int getBindIPv4Address() { + return lineUdpBindIPV4Address; + } + + @Override + public int getCommitRate() { + return lineUdpCommitRate; + } + + @Override + public int getGroupIPv4Address() { + return lineUdpGroupIPv4Address; + } + + @Override + public int getMsgBufferSize() { + return lineUdpMsgBufferSize; + } + + @Override + public int getMsgCount() { + return lineUdpMsgCount; + } + + @Override + public NetworkFacade getNetworkFacade() { + return NetworkFacadeImpl.INSTANCE; + } + + @Override + public int getPort() { + return lineUdpPort; + } + + @Override + public int getReceiveBufferSize() { + return lineUdpReceiveBufferSize; + } + } +} diff --git a/core/src/main/java/com/questdb/ServerConfigurationException.java b/core/src/main/java/com/questdb/ServerConfigurationException.java new file mode 100644 index 0000000000000000000000000000000000000000..82e637ca65188fa76683cbc1616e80964b693705 --- /dev/null +++ b/core/src/main/java/com/questdb/ServerConfigurationException.java @@ -0,0 +1,30 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (C) 2014-2019 Appsicle + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + ******************************************************************************/ + +package com.questdb; + +public class ServerConfigurationException extends Exception { + public ServerConfigurationException(String key, String value) { + super("invalid configuration value [key=" + key + ", value=" + value + "]"); + } +} diff --git a/core/src/main/java/com/questdb/ServerConfigurationV2.java b/core/src/main/java/com/questdb/ServerConfigurationV2.java new file mode 100644 index 0000000000000000000000000000000000000000..eab79e7b9b6356dd578b27b5912eccdd20b0e3cd --- /dev/null +++ b/core/src/main/java/com/questdb/ServerConfigurationV2.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (C) 2014-2019 Appsicle + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + ******************************************************************************/ + +package com.questdb; + +import com.questdb.cairo.CairoConfiguration; +import com.questdb.cutlass.http.HttpServerConfiguration; +import com.questdb.cutlass.line.udp.LineUdpReceiverConfiguration; + +public interface ServerConfigurationV2 { + + CairoConfiguration getCairoConfiguration(); + + HttpServerConfiguration getHttpServerConfiguration(); + + LineUdpReceiverConfiguration getLineUdpReceiverConfiguration(); +} diff --git a/core/src/main/java/com/questdb/cairo/CairoConfiguration.java b/core/src/main/java/com/questdb/cairo/CairoConfiguration.java index d7b2a32f5b495970aa27063baee0d22bfec335af..08a440f4e2fa61e9b9afcecc8c4062ff71dd1ee6 100644 --- a/core/src/main/java/com/questdb/cairo/CairoConfiguration.java +++ b/core/src/main/java/com/questdb/cairo/CairoConfiguration.java @@ -49,7 +49,7 @@ public interface CairoConfiguration { int getIndexValueBlockSize(); - int getMaxNumberOfSwapFiles(); + int getMaxSwapFileCount(); MicrosecondClock getMicrosecondClock(); @@ -59,19 +59,19 @@ public interface CairoConfiguration { int getParallelIndexThreshold(); - int getReaderPoolSegments(); + int getReaderPoolMaxSegments(); CharSequence getRoot(); long getSpinLockTimeoutUs(); - int getSqlCacheBlockCount(); + int getSqlCacheBlocks(); - int getSqlCacheBlockSize(); + int getSqlCacheRows(); int getSqlCharacterStoreCapacity(); - int getSqlCharacterStorePoolCapacity(); + int getSqlCharacterStoreSequencePoolCapacity(); int getSqlColumnPoolCapacity(); diff --git a/core/src/main/java/com/questdb/cairo/DefaultCairoConfiguration.java b/core/src/main/java/com/questdb/cairo/DefaultCairoConfiguration.java index 524f4d9dc76fae05887fb99f2889780563df64e9..ee29cc14ea859182e50b9bc081ab717989e2cfb7 100644 --- a/core/src/main/java/com/questdb/cairo/DefaultCairoConfiguration.java +++ b/core/src/main/java/com/questdb/cairo/DefaultCairoConfiguration.java @@ -91,7 +91,7 @@ public class DefaultCairoConfiguration implements CairoConfiguration { } @Override - public int getMaxNumberOfSwapFiles() { + public int getMaxSwapFileCount() { return 30; } @@ -116,7 +116,7 @@ public class DefaultCairoConfiguration implements CairoConfiguration { } @Override - public int getReaderPoolSegments() { + public int getReaderPoolMaxSegments() { return 5; } @@ -131,13 +131,13 @@ public class DefaultCairoConfiguration implements CairoConfiguration { } @Override - public int getSqlCacheBlockCount() { - return 16; + public int getSqlCacheBlocks() { + return 4; } @Override - public int getSqlCacheBlockSize() { - return 4; + public int getSqlCacheRows() { + return 16; } @Override @@ -148,7 +148,7 @@ public class DefaultCairoConfiguration implements CairoConfiguration { } @Override - public int getSqlCharacterStorePoolCapacity() { + public int getSqlCharacterStoreSequencePoolCapacity() { return 64; } diff --git a/core/src/main/java/com/questdb/cairo/TableUtils.java b/core/src/main/java/com/questdb/cairo/TableUtils.java index d3422040094308b79f53db8c4d81934e5bd141c6..80443391246301ce79b1a1cb7c0c46f3fb618717 100644 --- a/core/src/main/java/com/questdb/cairo/TableUtils.java +++ b/core/src/main/java/com/questdb/cairo/TableUtils.java @@ -98,7 +98,7 @@ public final class TableUtils { FilesFacade ff, AppendMemory memory, Path path, - CharSequence root, + @Transient CharSequence root, TableStructure structure, int mkDirMode ) { diff --git a/core/src/main/java/com/questdb/cairo/TableWriter.java b/core/src/main/java/com/questdb/cairo/TableWriter.java index 9bb9cae126370532cfbf07f55d92e6c08e6c411c..f20a86a6f4e9e2ead94aa453745b612cc2a8aec7 100644 --- a/core/src/main/java/com/questdb/cairo/TableWriter.java +++ b/core/src/main/java/com/questdb/cairo/TableWriter.java @@ -814,7 +814,7 @@ public class TableWriter implements Closeable { private int addColumnToMeta(CharSequence name, int type, boolean indexFlag, int indexValueBlockCapacity) { int index; try { - index = openMetaSwapFile(ff, ddlMem, path, rootLen, configuration.getMaxNumberOfSwapFiles()); + index = openMetaSwapFile(ff, ddlMem, path, rootLen, configuration.getMaxSwapFileCount()); int columnCount = metaMem.getInt(META_OFFSET_COUNT); ddlMem.putInt(columnCount + 1); diff --git a/core/src/main/java/com/questdb/cairo/pool/ReaderPool.java b/core/src/main/java/com/questdb/cairo/pool/ReaderPool.java index 28aed20bc67911cbc0926ef18ff37303f6ea7843..b75eb7ab94c9239569448bad51d1e0179c551df9 100644 --- a/core/src/main/java/com/questdb/cairo/pool/ReaderPool.java +++ b/core/src/main/java/com/questdb/cairo/pool/ReaderPool.java @@ -53,7 +53,7 @@ public class ReaderPool extends AbstractPool implements ResourcePool(DirectByteCharSequence.FACTORY, configuration.getConnectionStringPoolSize()); + this.csPool = new ObjectPool<>(DirectByteCharSequence.FACTORY, configuration.getConnectionStringPoolCapacity()); this.headerParser = new HttpHeaderParser(configuration.getRequestHeaderBufferSize(), csPool); this.multipartContentHeaderParser = new HttpHeaderParser(configuration.getMultipartHeaderBufferSize(), csPool); this.multipartContentParser = new HttpMultipartContentParser(multipartContentHeaderParser); diff --git a/core/src/main/java/com/questdb/cutlass/http/HttpServer.java b/core/src/main/java/com/questdb/cutlass/http/HttpServer.java index aa5730f4597ae132eceadf33aed00056a53d5097..0bd0178da5fe6064f5c156aa3bda89b9212a85bf 100644 --- a/core/src/main/java/com/questdb/cutlass/http/HttpServer.java +++ b/core/src/main/java/com/questdb/cutlass/http/HttpServer.java @@ -145,7 +145,7 @@ public class HttpServer implements Closeable { workerHaltLatch, -1, LOG, - configuration.getConnectionPoolInitialSize(), + configuration.getConnectionPoolInitialCapacity(), // have each thread release their own processor selectors // in case processors stash some of their resources in thread-local variables () -> Misc.free(selectors.getQuick(index)) diff --git a/core/src/main/java/com/questdb/cutlass/http/HttpServerConfiguration.java b/core/src/main/java/com/questdb/cutlass/http/HttpServerConfiguration.java index af174167bd4e141f6e78bd5ff3a875a42f5ba6a7..5e140d1092787d1e194ae3a76194eba8689a03be 100644 --- a/core/src/main/java/com/questdb/cutlass/http/HttpServerConfiguration.java +++ b/core/src/main/java/com/questdb/cutlass/http/HttpServerConfiguration.java @@ -31,9 +31,9 @@ import com.questdb.std.time.MillisecondClock; public interface HttpServerConfiguration { String DEFAULT_PROCESSOR_URL = "*"; - int getConnectionPoolInitialSize(); + int getConnectionPoolInitialCapacity(); - int getConnectionStringPoolSize(); + int getConnectionStringPoolCapacity(); int getMultipartHeaderBufferSize(); diff --git a/core/src/main/java/com/questdb/cutlass/http/PropHttpServerConfiguration.java b/core/src/main/java/com/questdb/cutlass/http/PropHttpServerConfiguration.java deleted file mode 100644 index cb5d080fa5190dbaea1020c52ac1977757b4045e..0000000000000000000000000000000000000000 --- a/core/src/main/java/com/questdb/cutlass/http/PropHttpServerConfiguration.java +++ /dev/null @@ -1,372 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (C) 2014-2019 Appsicle - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - ******************************************************************************/ - -package com.questdb.cutlass.http; - -import com.questdb.cutlass.http.processors.StaticContentProcessorConfiguration; -import com.questdb.cutlass.http.processors.TextImportProcessorConfiguration; -import com.questdb.cutlass.text.TextConfiguration; -import com.questdb.network.*; -import com.questdb.std.FilesFacade; -import com.questdb.std.FilesFacadeImpl; -import com.questdb.std.Numbers; -import com.questdb.std.NumericException; -import com.questdb.std.time.MillisecondClock; -import com.questdb.std.time.MillisecondClockImpl; - -import java.util.Properties; - -public class PropHttpServerConfiguration implements HttpServerConfiguration { - - private final IODispatcherConfiguration ioDispatcherConfiguration = new PropIODispatcherConfiguration(); - private final TextImportProcessorConfiguration textImportProcessorConfiguration = new PropTextImportProcessorConfiguration(); - private final StaticContentProcessorConfiguration staticContentProcessorConfiguration = new PropStaticContentProcessorConfiguration(); - private final PropTextConfiguration textConfiguration = new PropTextConfiguration(); - private final int connectionPoolInitialSize; - private final int connectionStringPoolSize; - private final int multipartHeaderBufferSize; - private final long multipartIdleSpinCount; - private final int recvBufferSize; - private final int requestHeaderBufferSize; - private final int responseHeaderBufferSize; - private final int workerCount; - private final int sendBufferSize; - private final CharSequence indexFileName; - private final CharSequence publicDirectory; - private final boolean abortBrokenUploads; - private final int activeConnectionLimit; - private final CharSequence bindIPv4Address; - private final int bindPort; - private final int eventCapacity; - private final int ioQueueCapacity; - private final long idleConnectionTimeout; - private final int interestQueueCapacity; - private final int listenBacklog; - private final int sndBufSize; - private final int rcvBufSize; - private final String adapterSetConfigurationFileName; - private final int dateAdapterPoolSize; - private final int jsonCacheLimit; - private final int jsonCacheSize; - private final double maxRequiredDelimiterStdDev; - private final int metadataStringPoolSize; - private final long rollBufferLimit; - private final long rollBufferSize; - private final int textAnalysisMaxLines; - private final int textLexerStringPoolSize; - private final int timestampAdapterPoolSize; - private final int utf8SinkCapacity; - - public PropHttpServerConfiguration(Properties properties) { - this.connectionPoolInitialSize = getInt(properties, "http.connection.pool.initial.size"); - this.connectionStringPoolSize = getInt(properties, "http.connection.string.pool.size"); - this.multipartHeaderBufferSize = getInt(properties, "http.multipart.header.buffer.size"); - this.multipartIdleSpinCount = Long.parseLong(properties.getProperty("http.multipart.idle.spin.count")); - this.recvBufferSize = getInt(properties, "http.receive.buffer.size"); - this.requestHeaderBufferSize = getInt(properties, "http.request.header.buffer.size"); - this.responseHeaderBufferSize = getInt(properties, "http.response.header.buffer.size"); - this.workerCount = getInt(properties, "http.worker.count"); - this.sendBufferSize = getInt(properties, "http.send.buffer.size"); - this.indexFileName = getString(properties, "http.static.index.file.name"); - this.publicDirectory = getString(properties, "http.static.pubic.directory"); - this.abortBrokenUploads = Boolean.parseBoolean(properties.getProperty("http.text.abort.broken.uploads")); - this.activeConnectionLimit = getInt(properties, "http.net.active.connection.limit"); - this.eventCapacity = getInt(properties, "http.net.event.capacity"); - this.ioQueueCapacity = getInt(properties, "http.net.io.queue.capacity"); - this.idleConnectionTimeout = Long.parseLong(properties.getProperty("http.net.idle.connection.timeout")); - this.interestQueueCapacity = getInt(properties, "http.net.interest.queue.capacity"); - this.listenBacklog = getInt(properties, "http.net.listen.backlog"); - this.sndBufSize = getInt(properties, "http.net.snd.buf.size"); - this.rcvBufSize = getInt(properties, "http.net.rcv.buf.size"); - this.adapterSetConfigurationFileName = getString(properties, "http.text.adapter.set.config"); - this.dateAdapterPoolSize = getInt(properties, "http.text.date.adapter.pool.size"); - this.jsonCacheLimit = getInt(properties, "http.text.json.cache.limit"); - this.jsonCacheSize = getInt(properties, "http.text.json.cache.size"); - this.maxRequiredDelimiterStdDev = Double.parseDouble(properties.getProperty("http.text.max.required.delimiter.stddev")); - this.metadataStringPoolSize = getInt(properties, "http.text.metadata.string.pool.size"); - this.rollBufferLimit = Long.parseLong(properties.getProperty("http.text.roll.buffer.limit")); - this.rollBufferSize = Long.getLong(properties.getProperty("http.text.roll.buffer.size")); - this.textAnalysisMaxLines = getInt(properties, "http.text.analysis.max.lines"); - this.textLexerStringPoolSize = getInt(properties, "http.text.lexer.string.pool.size"); - this.timestampAdapterPoolSize = getInt(properties, "http.text.timestamp.adapter.pool.size"); - this.utf8SinkCapacity = getInt(properties, "http.text.utf8.sink.capacity"); - final String httpBindTo = getString(properties, "http.bind.to"); - final int colonIndex = httpBindTo.indexOf(':'); - if (colonIndex == -1) { - throw new IllegalStateException(); - } - this.bindIPv4Address = httpBindTo.substring(0, colonIndex); - this.bindPort = Integer.parseInt(httpBindTo.substring(colonIndex + 1)); - - } - - @Override - public int getConnectionPoolInitialSize() { - return connectionPoolInitialSize; - } - - @Override - public int getConnectionStringPoolSize() { - return connectionStringPoolSize; - } - - @Override - public int getMultipartHeaderBufferSize() { - return multipartHeaderBufferSize; - } - - @Override - public long getMultipartIdleSpinCount() { - return multipartIdleSpinCount; - } - - @Override - public int getRecvBufferSize() { - return recvBufferSize; - } - - @Override - public int getRequestHeaderBufferSize() { - return requestHeaderBufferSize; - } - - @Override - public int getResponseHeaderBufferSize() { - return responseHeaderBufferSize; - } - - @Override - public MillisecondClock getClock() { - return MillisecondClockImpl.INSTANCE; - } - - @Override - public IODispatcherConfiguration getDispatcherConfiguration() { - return ioDispatcherConfiguration; - } - - @Override - public StaticContentProcessorConfiguration getStaticContentProcessorConfiguration() { - return staticContentProcessorConfiguration; - } - - @Override - public TextImportProcessorConfiguration getTextImportProcessorConfiguration() { - return textImportProcessorConfiguration; - } - - @Override - public int getWorkerCount() { - return workerCount; - } - - @Override - public int getSendBufferSize() { - return sendBufferSize; - } - - private int getInt(Properties properties, String key) { - try { - return Numbers.parseInt(properties.getProperty(key)); - } catch (NumericException e) { - return 0; - } - } - - private String getString(Properties properties, String key) { - return properties.getProperty(key); - } - - private class PropStaticContentProcessorConfiguration implements StaticContentProcessorConfiguration { - @Override - public FilesFacade getFilesFacade() { - return FilesFacadeImpl.INSTANCE; - } - - @Override - public CharSequence getIndexFileName() { - return indexFileName; - } - - @Override - public MimeTypesCache getMimeTypesCache() { - return null; - } - - @Override - public CharSequence getPublicDirectory() { - return publicDirectory; - } - } - - private class PropTextImportProcessorConfiguration implements TextImportProcessorConfiguration { - @Override - public boolean abortBrokenUploads() { - return abortBrokenUploads; - } - - @Override - public TextConfiguration getTextConfiguration() { - return textConfiguration; - } - } - - private class PropIODispatcherConfiguration implements IODispatcherConfiguration { - @Override - public int getActiveConnectionLimit() { - return activeConnectionLimit; - } - - @Override - public CharSequence getBindIPv4Address() { - return bindIPv4Address; - } - - @Override - public int getBindPort() { - return bindPort; - } - - @Override - public MillisecondClock getClock() { - return MillisecondClockImpl.INSTANCE; - } - - @Override - public int getEventCapacity() { - return eventCapacity; - } - - @Override - public int getIOQueueCapacity() { - return ioQueueCapacity; - } - - @Override - public long getIdleConnectionTimeout() { - return idleConnectionTimeout; - } - - @Override - public int getInterestQueueCapacity() { - return interestQueueCapacity; - } - - @Override - public int getListenBacklog() { - return listenBacklog; - } - - @Override - public NetworkFacade getNetworkFacade() { - return NetworkFacadeImpl.INSTANCE; - } - - @Override - public EpollFacade getEpollFacade() { - return EpollFacadeImpl.INSTANCE; - } - - @Override - public SelectFacade getSelectFacade() { - return SelectFacadeImpl.INSTANCE; - } - - @Override - public int getInitialBias() { - return IOOperation.READ; - } - - @Override - public int getSndBufSize() { - return sndBufSize; - } - - @Override - public int getRcvBufSize() { - return rcvBufSize; - } - } - - private class PropTextConfiguration implements TextConfiguration { - @Override - public String getAdapterSetConfigurationFileName() { - return adapterSetConfigurationFileName; - } - - @Override - public int getDateAdapterPoolSize() { - return dateAdapterPoolSize; - } - - @Override - public int getJsonCacheLimit() { - return jsonCacheLimit; - } - - @Override - public int getJsonCacheSize() { - return jsonCacheSize; - } - - @Override - public double getMaxRequiredDelimiterStdDev() { - return maxRequiredDelimiterStdDev; - } - - @Override - public int getMetadataStringPoolSize() { - return metadataStringPoolSize; - } - - @Override - public long getRollBufferLimit() { - return rollBufferLimit; - } - - @Override - public long getRollBufferSize() { - return rollBufferSize; - } - - @Override - public int getTextAnalysisMaxLines() { - return textAnalysisMaxLines; - } - - @Override - public int getTextLexerStringPoolSize() { - return textLexerStringPoolSize; - } - - @Override - public int getTimestampAdapterPoolSize() { - return timestampAdapterPoolSize; - } - - @Override - public int getUtf8SinkCapacity() { - return utf8SinkCapacity; - } - } -} diff --git a/core/src/main/java/com/questdb/cutlass/http/processors/JsonQueryProcessor.java b/core/src/main/java/com/questdb/cutlass/http/processors/JsonQueryProcessor.java index 6f3bcfdf6168704b271524d2164ec206b8b7d170..3f271268734191385427b1dc788cb69327b29aaa 100644 --- a/core/src/main/java/com/questdb/cutlass/http/processors/JsonQueryProcessor.java +++ b/core/src/main/java/com/questdb/cutlass/http/processors/JsonQueryProcessor.java @@ -53,7 +53,7 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { static final int QUERY_METADATA = 2; static final int QUERY_PREFIX = 1; // Factory cache is thread local due to possibility of factory being - // close by another thread. Peer disconnect is a typical example of this. + // closed by another thread. Peer disconnect is a typical example of this. // Being asynchronous we may need to be able to return factory to the cache // by the same thread that executes the dispatcher. static final ThreadLocal> FACTORY_CACHE = ThreadLocal.withInitial(() -> new AssociativeCache<>(8, 8)); @@ -323,9 +323,6 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { HttpChunkedResponseSocket socket, int status ) throws PeerDisconnectedException, PeerIsSlowToReadException { - // todo: same issue as with sending confirmation - // what happens when header doesn't fit in buffer or there is "slow" client - // half way sending header socket.status(status, "application/json; charset=utf-8"); // todo: configure this header externally socket.headers().put("Keep-Alive: timeout=5, max=10000").put(Misc.EOL); diff --git a/core/src/main/java/com/questdb/cutlass/json/JsonLexer.java b/core/src/main/java/com/questdb/cutlass/json/JsonLexer.java index d51221bcc40b1d824094a061ed3bc05b77c078b0..6abb9196d3bb233fdda0cca6025afd9b7a5f5dd6 100644 --- a/core/src/main/java/com/questdb/cutlass/json/JsonLexer.java +++ b/core/src/main/java/com/questdb/cutlass/json/JsonLexer.java @@ -61,9 +61,9 @@ public class JsonLexer implements Mutable, Closeable { private boolean useCache = false; private int position = 0; - public JsonLexer(int cacheCapacity, int cacheSizeLimit) { - this.cacheCapacity = cacheCapacity; - this.cache = Unsafe.malloc(cacheCapacity); + public JsonLexer(int cacheSize, int cacheSizeLimit) { + this.cacheCapacity = cacheSize; + this.cache = Unsafe.malloc(cacheSize); this.cacheSizeLimit = cacheSizeLimit; } diff --git a/core/src/main/java/com/questdb/cutlass/line/CairoLineProtoParser.java b/core/src/main/java/com/questdb/cutlass/line/CairoLineProtoParser.java index 88ee566b01e27b6b50731a4e75de63fd3847a823..1647b733e29eff484a5dcb4821e3cb9e3fe206a1 100644 --- a/core/src/main/java/com/questdb/cutlass/line/CairoLineProtoParser.java +++ b/core/src/main/java/com/questdb/cutlass/line/CairoLineProtoParser.java @@ -24,7 +24,7 @@ package com.questdb.cutlass.line; import com.questdb.cairo.*; -import com.questdb.cairo.pool.ResourcePool; +import com.questdb.cairo.sql.CairoEngine; import com.questdb.cairo.sql.RecordMetadata; import com.questdb.log.Log; import com.questdb.log.LogFactory; @@ -45,7 +45,7 @@ public class CairoLineProtoParser implements LineProtoParser, Closeable { }; private static final FieldNameParser NOOP_FIELD_NAME = name -> { }; - private final ResourcePool pool; + private final CairoEngine engine; private final CharSequenceObjHashMap writerCache = new CharSequenceObjHashMap<>(); private final CharSequenceObjHashMap commitList = new CharSequenceObjHashMap<>(); private final Path path = new Path(); @@ -78,10 +78,10 @@ public class CairoLineProtoParser implements LineProtoParser, Closeable { private final TableStructureAdapter tableStructureAdapter = new TableStructureAdapter(); private final LineEndParser MY_NEW_LINE_END = this::createTableAndAppendRow; - public CairoLineProtoParser(CairoConfiguration configuration, ResourcePool pool) { - this.configuration = configuration; + public CairoLineProtoParser(CairoEngine engine) { + this.configuration = engine.getConfiguration(); this.clock = configuration.getMicrosecondClock(); - this.pool = pool; + this.engine = engine; } @Override @@ -157,7 +157,7 @@ public class CairoLineProtoParser implements LineProtoParser, Closeable { } private void appendFirstRowAndCacheWriter(CharSequenceCache cache) { - TableWriter writer = pool.get(cache.get(tableName)); + TableWriter writer = engine.getWriter(cache.get(tableName)); this.writer = writer; this.metadata = writer.getMetadata(); this.columnCount = metadata.getColumnCount(); @@ -224,7 +224,7 @@ public class CairoLineProtoParser implements LineProtoParser, Closeable { private void cacheWriter(CacheEntry entry, CachedCharSequence tableName) { try { - entry.writer = pool.get(tableName); + entry.writer = engine.getWriter(tableName); this.tableName = tableName.getCacheAddress(); createState(entry); LOG.info().$("cached writer [name=").$(tableName).$(']').$(); @@ -251,7 +251,7 @@ public class CairoLineProtoParser implements LineProtoParser, Closeable { configuration.getFilesFacade(), appendMemory, path, - configuration.getRoot().toString(), + configuration.getRoot(), tableStructureAdapter.of(cache), configuration.getMkDirMode() ); diff --git a/core/src/main/java/com/questdb/cutlass/line/udp/GenericLineProtoReceiver.java b/core/src/main/java/com/questdb/cutlass/line/udp/GenericLineProtoReceiver.java index 0a57fb5feae3d506006d130cc96629b71f0bac7b..620bcb111551051c961ab504dd2d6f3b4da01ec6 100644 --- a/core/src/main/java/com/questdb/cutlass/line/udp/GenericLineProtoReceiver.java +++ b/core/src/main/java/com/questdb/cutlass/line/udp/GenericLineProtoReceiver.java @@ -23,10 +23,8 @@ package com.questdb.cutlass.line.udp; -import com.questdb.cairo.CairoConfiguration; import com.questdb.cairo.CairoException; -import com.questdb.cairo.TableWriter; -import com.questdb.cairo.pool.ResourcePool; +import com.questdb.cairo.sql.CairoEngine; import com.questdb.cutlass.line.CairoLineProtoParser; import com.questdb.cutlass.line.LineProtoLexer; import com.questdb.log.Log; @@ -52,7 +50,7 @@ public class GenericLineProtoReceiver implements Closeable, Job { private long totalCount = 0; private long buf; - public GenericLineProtoReceiver(ReceiverConfiguration receiverCfg, CairoConfiguration cairoCfg, ResourcePool writerPool) { + public GenericLineProtoReceiver(LineUdpReceiverConfiguration receiverCfg, CairoEngine engine) { nf = receiverCfg.getNetworkFacade(); @@ -89,7 +87,7 @@ public class GenericLineProtoReceiver implements Closeable, Job { this.buf = Unsafe.malloc(this.bufLen = receiverCfg.getMsgBufferSize()); lexer = new LineProtoLexer(receiverCfg.getMsgBufferSize()); - parser = new CairoLineProtoParser(cairoCfg, writerPool); + parser = new CairoLineProtoParser(engine); lexer.withParser(parser); LOG.info() diff --git a/core/src/main/java/com/questdb/cutlass/line/udp/ReceiverConfiguration.java b/core/src/main/java/com/questdb/cutlass/line/udp/LineUdpReceiverConfiguration.java similarity index 91% rename from core/src/main/java/com/questdb/cutlass/line/udp/ReceiverConfiguration.java rename to core/src/main/java/com/questdb/cutlass/line/udp/LineUdpReceiverConfiguration.java index 181efcc8df56b6dd8a85555549a55a968ad08ac4..9d65ca0b99a259a52b7f551ce7bb271e0459246b 100644 --- a/core/src/main/java/com/questdb/cutlass/line/udp/ReceiverConfiguration.java +++ b/core/src/main/java/com/questdb/cutlass/line/udp/LineUdpReceiverConfiguration.java @@ -25,13 +25,13 @@ package com.questdb.cutlass.line.udp; import com.questdb.network.NetworkFacade; -public interface ReceiverConfiguration { +public interface LineUdpReceiverConfiguration { - CharSequence getBindIPv4Address(); + int getBindIPv4Address(); int getCommitRate(); - CharSequence getGroupIPv4Address(); + int getGroupIPv4Address(); int getMsgBufferSize(); diff --git a/core/src/main/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiver.java b/core/src/main/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiver.java index 1c46dabd98e104a7f5ccbec382a08bffcd00fbb7..9f44b2a45d8b9d461ecd417d9ed55fb597fad994 100644 --- a/core/src/main/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiver.java +++ b/core/src/main/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiver.java @@ -23,10 +23,8 @@ package com.questdb.cutlass.line.udp; -import com.questdb.cairo.CairoConfiguration; import com.questdb.cairo.CairoException; -import com.questdb.cairo.TableWriter; -import com.questdb.cairo.pool.ResourcePool; +import com.questdb.cairo.sql.CairoEngine; import com.questdb.cutlass.line.CairoLineProtoParser; import com.questdb.cutlass.line.LineProtoLexer; import com.questdb.log.Log; @@ -50,7 +48,7 @@ public class LinuxLineProtoReceiver implements Closeable, Job { private int commitRate; private long totalCount = 0; - public LinuxLineProtoReceiver(ReceiverConfiguration receiverCfg, CairoConfiguration cairoCfg, ResourcePool writerPool) { + public LinuxLineProtoReceiver(LineUdpReceiverConfiguration receiverCfg, CairoEngine engine) { nf = receiverCfg.getNetworkFacade(); @@ -87,7 +85,7 @@ public class LinuxLineProtoReceiver implements Closeable, Job { msgVec = nf.msgHeaders(receiverCfg.getMsgBufferSize(), msgCount); lexer = new LineProtoLexer(receiverCfg.getMsgBufferSize()); - parser = new CairoLineProtoParser(cairoCfg, writerPool); + parser = new CairoLineProtoParser(engine); lexer.withParser(parser); LOG.info().$("started [fd=").$(fd).$(", bind=").$(receiverCfg.getBindIPv4Address()).$(", group=").$(receiverCfg.getGroupIPv4Address()).$(", port=").$(receiverCfg.getPort()).$(", batch=").$(msgCount).$(", commitRate=").$(commitRate).$(']').$(); diff --git a/core/src/main/java/com/questdb/cutlass/line/udp/ReceiverFactory.java b/core/src/main/java/com/questdb/cutlass/line/udp/ReceiverFactory.java index 7d045afd9650c516778e1e4241df422617dfafec..5a3c31e1acbe44f5f5e9cef9324812b115ad3fc5 100644 --- a/core/src/main/java/com/questdb/cutlass/line/udp/ReceiverFactory.java +++ b/core/src/main/java/com/questdb/cutlass/line/udp/ReceiverFactory.java @@ -23,11 +23,10 @@ package com.questdb.cutlass.line.udp; -import com.questdb.cairo.CairoConfiguration; -import com.questdb.cairo.pool.WriterPool; +import com.questdb.cairo.sql.CairoEngine; import com.questdb.mp.Job; @FunctionalInterface public interface ReceiverFactory { - Job createReceiver(ReceiverConfiguration receiverCfg, CairoConfiguration cairoCfg, WriterPool pool); + Job createReceiver(LineUdpReceiverConfiguration receiverCfg, CairoEngine engine); } diff --git a/core/src/main/java/com/questdb/cutlass/text/CairoTextWriter.java b/core/src/main/java/com/questdb/cutlass/text/CairoTextWriter.java index 911e5afb7deac277f5940b71d4726922ef61c6a2..121fc789c5a91dd8efd5f2d553104ce6ca4600b6 100644 --- a/core/src/main/java/com/questdb/cutlass/text/CairoTextWriter.java +++ b/core/src/main/java/com/questdb/cutlass/text/CairoTextWriter.java @@ -70,7 +70,7 @@ public class CairoTextWriter implements TextLexer.Listener, Closeable, Mutable { this.engine = engine; this.configuration = engine.getConfiguration(); this.path = path; - this.utf8Sink = new DirectCharSink(textConfiguration.getUtf8SinkCapacity()); + this.utf8Sink = new DirectCharSink(textConfiguration.getUtf8SinkSize()); this.typeManager = typeManager; } diff --git a/core/src/main/java/com/questdb/cutlass/text/DefaultTextConfiguration.java b/core/src/main/java/com/questdb/cutlass/text/DefaultTextConfiguration.java index e9f073c1242eb92e7de5d694fc7a85b14adf7fef..313e205ab3928ea83261190d7f93253532540f95 100644 --- a/core/src/main/java/com/questdb/cutlass/text/DefaultTextConfiguration.java +++ b/core/src/main/java/com/questdb/cutlass/text/DefaultTextConfiguration.java @@ -30,13 +30,13 @@ public class DefaultTextConfiguration implements TextConfiguration { } @Override - public int getDateAdapterPoolSize() { + public int getDateAdapterPoolCapacity() { return 16; } @Override public int getJsonCacheLimit() { - return 4096; + return 16384; } @Override @@ -50,17 +50,17 @@ public class DefaultTextConfiguration implements TextConfiguration { } @Override - public int getMetadataStringPoolSize() { + public int getMetadataStringPoolCapacity() { return 128; } @Override - public long getRollBufferLimit() { + public int getRollBufferLimit() { return 4096; } @Override - public long getRollBufferSize() { + public int getRollBufferSize() { return 1024; } @@ -70,17 +70,17 @@ public class DefaultTextConfiguration implements TextConfiguration { } @Override - public int getTextLexerStringPoolSize() { + public int getTextLexerStringPoolCapacity() { return 32; } @Override - public int getTimestampAdapterPoolSize() { + public int getTimestampAdapterPoolCapacity() { return 16; } @Override - public int getUtf8SinkCapacity() { + public int getUtf8SinkSize() { return 4096; } } diff --git a/core/src/main/java/com/questdb/cutlass/text/TextConfiguration.java b/core/src/main/java/com/questdb/cutlass/text/TextConfiguration.java index d8e494b86bbed947a2303e3bc08cd077acb2f023..bfd1cedb200643ae877e776fe86fd67525f6f4af 100644 --- a/core/src/main/java/com/questdb/cutlass/text/TextConfiguration.java +++ b/core/src/main/java/com/questdb/cutlass/text/TextConfiguration.java @@ -26,7 +26,7 @@ package com.questdb.cutlass.text; public interface TextConfiguration { String getAdapterSetConfigurationFileName(); - int getDateAdapterPoolSize(); + int getDateAdapterPoolCapacity(); int getJsonCacheLimit(); @@ -34,17 +34,17 @@ public interface TextConfiguration { double getMaxRequiredDelimiterStdDev(); - int getMetadataStringPoolSize(); + int getMetadataStringPoolCapacity(); - long getRollBufferLimit(); + int getRollBufferLimit(); - long getRollBufferSize(); + int getRollBufferSize(); int getTextAnalysisMaxLines(); - int getTextLexerStringPoolSize(); + int getTextLexerStringPoolCapacity(); - int getTimestampAdapterPoolSize(); + int getTimestampAdapterPoolCapacity(); - int getUtf8SinkCapacity(); + int getUtf8SinkSize(); } diff --git a/core/src/main/java/com/questdb/cutlass/text/TextLexer.java b/core/src/main/java/com/questdb/cutlass/text/TextLexer.java index 612ffbf6596b4bb00c8230d0cd6b5de7620bb86d..9cc23c9d3f2ad942abb5a0b7217840b3d61a1cea 100644 --- a/core/src/main/java/com/questdb/cutlass/text/TextLexer.java +++ b/core/src/main/java/com/questdb/cutlass/text/TextLexer.java @@ -41,7 +41,7 @@ public class TextLexer implements Closeable, Mutable { private final ObjList fields = new ObjList<>(); private final ObjectPool csPool; private final TextMetadataDetector metadataDetector; - private final long lineRollBufLimit; + private final int lineRollBufLimit; private boolean ignoreEolOnce; private byte columnDelimiter; private boolean inQuote; @@ -56,7 +56,7 @@ public class TextLexer implements Closeable, Mutable { private long lineRollBufCur; private Listener textLexerListener; private long lastLineStart; - private long lineRollBufLen; + private int lineRollBufLen; private long lineRollBufPtr; private boolean header; private long lastQuotePos = -1; @@ -66,7 +66,7 @@ public class TextLexer implements Closeable, Mutable { public TextLexer(TextConfiguration textConfiguration, TypeManager typeManager) { this.metadataDetector = new TextMetadataDetector(typeManager, textConfiguration); - this.csPool = new ObjectPool<>(DirectByteCharSequence.FACTORY, textConfiguration.getTextLexerStringPoolSize()); + this.csPool = new ObjectPool<>(DirectByteCharSequence.FACTORY, textConfiguration.getTextLexerStringPoolCapacity()); this.lineRollBufLen = textConfiguration.getRollBufferSize(); this.lineRollBufLimit = textConfiguration.getRollBufferLimit(); this.lineRollBufPtr = Unsafe.malloc(lineRollBufLen); @@ -164,7 +164,7 @@ public class TextLexer implements Closeable, Mutable { return metadataDetector.getColumnTypes(); } - private boolean growRollBuf(long requiredLength) { + private boolean growRollBuf(int requiredLength) { if (requiredLength > lineRollBufLimit) { // todo: log content of roll buffer LOG.info().$("too long [table=").$(tableName).$(", line=").$(lineCount).$(']').$(); @@ -173,7 +173,7 @@ public class TextLexer implements Closeable, Mutable { return false; } - final long len = Math.min(lineRollBufLimit, requiredLength << 1); + final int len = Math.min(lineRollBufLimit, requiredLength << 1); LOG.info().$("resizing ").$(lineRollBufLen).$(" -> ").$(len).$(" [table=").$(tableName).$(']').$(); long p = Unsafe.malloc(len); long l = lineRollBufCur - lineRollBufPtr; @@ -327,7 +327,7 @@ public class TextLexer implements Closeable, Mutable { private void rollLine(long lo, long hi) { // lastLineStart is an offset from 'lo' // 'lo' is the address of incoming buffer - long l = hi - lo - lastLineStart; + int l = (int) (hi - lo - lastLineStart); if (l < lineRollBufLen || growRollBuf(l)) { assert lo + lastLineStart + l <= hi; Unsafe.getUnsafe().copyMemory(lo + lastLineStart, lineRollBufPtr, l); diff --git a/core/src/main/java/com/questdb/cutlass/text/TextLoader.java b/core/src/main/java/com/questdb/cutlass/text/TextLoader.java index c6ba684679fb47610ef191cbff95628b85724e69..9784076a6fa5750c3617b34d8199fed6132436e6 100644 --- a/core/src/main/java/com/questdb/cutlass/text/TextLoader.java +++ b/core/src/main/java/com/questdb/cutlass/text/TextLoader.java @@ -67,7 +67,7 @@ public class TextLoader implements Closeable, Mutable { com.questdb.std.microtime.DateFormatFactory timestampFormatFactory ) throws JsonException { - this.utf8Sink = new DirectCharSink(textConfiguration.getUtf8SinkCapacity()); + this.utf8Sink = new DirectCharSink(textConfiguration.getUtf8SinkSize()); jsonLexer = new JsonLexer( textConfiguration.getJsonCacheSize(), textConfiguration.getJsonCacheLimit() diff --git a/core/src/main/java/com/questdb/cutlass/text/TextMetadataDetector.java b/core/src/main/java/com/questdb/cutlass/text/TextMetadataDetector.java index 4bf6f4f75ebb6deae0876cb2037d424018d76620..9d9739f1e4d60261eee7e3970b1aa4d75bf5b7f9 100644 --- a/core/src/main/java/com/questdb/cutlass/text/TextMetadataDetector.java +++ b/core/src/main/java/com/questdb/cutlass/text/TextMetadataDetector.java @@ -58,7 +58,7 @@ public class TextMetadataDetector implements TextLexer.Listener, Mutable, Closea TextConfiguration textConfiguration ) { this.typeManager = typeManager; - this.utf8Sink = new DirectCharSink(textConfiguration.getUtf8SinkCapacity()); + this.utf8Sink = new DirectCharSink(textConfiguration.getUtf8SinkSize()); } public static boolean utf8Decode(long lo, long hi, CharSink sink) { diff --git a/core/src/main/java/com/questdb/cutlass/text/TextMetadataParser.java b/core/src/main/java/com/questdb/cutlass/text/TextMetadataParser.java index 17b42e5554be9c8a69c4e9fa27d982ae926dd556..da9cdd477d50ad94d687f18d74b6cacd4d0e6690 100644 --- a/core/src/main/java/com/questdb/cutlass/text/TextMetadataParser.java +++ b/core/src/main/java/com/questdb/cutlass/text/TextMetadataParser.java @@ -80,7 +80,7 @@ public class TextMetadataParser implements JsonParser, Mutable, Closeable { ) { this.columnNames = new ObjList<>(); this.columnTypes = new ObjList<>(); - this.csPool = new ObjectPool<>(FloatingCharSequence::new, textConfiguration.getMetadataStringPoolSize()); + this.csPool = new ObjectPool<>(FloatingCharSequence::new, textConfiguration.getMetadataStringPoolCapacity()); this.dateLocaleFactory = dateLocaleFactory; this.dateFormatFactory = dateFormatFactory; this.timestampLocaleFactory = timestampLocaleFactory; diff --git a/core/src/main/java/com/questdb/cutlass/text/types/TypeManager.java b/core/src/main/java/com/questdb/cutlass/text/types/TypeManager.java index 913c5745cd895605aa9622866092293c4a11c5e0..0eca7b1335d56d6436e9dc7e2c8667638dec6419 100644 --- a/core/src/main/java/com/questdb/cutlass/text/types/TypeManager.java +++ b/core/src/main/java/com/questdb/cutlass/text/types/TypeManager.java @@ -72,8 +72,8 @@ public class TypeManager implements Mutable { public TypeManager(TextConfiguration configuration, DirectCharSink utf8Sink, JsonLexer jsonLexer) throws JsonException { this.utf8Sink = utf8Sink; - this.dateAdapterPool = new ObjectPool<>(() -> new DateAdapter(utf8Sink), configuration.getDateAdapterPoolSize()); - this.timestampAdapterPool = new ObjectPool<>(() -> new TimestampAdapter(utf8Sink), configuration.getTimestampAdapterPoolSize()); + this.dateAdapterPool = new ObjectPool<>(() -> new DateAdapter(utf8Sink), configuration.getDateAdapterPoolCapacity()); + this.timestampAdapterPool = new ObjectPool<>(() -> new TimestampAdapter(utf8Sink), configuration.getTimestampAdapterPoolCapacity()); this.stringAdapter = new StringAdapter(utf8Sink); this.symbolAdapter = new SymbolAdapter(utf8Sink); this.jsonLexer = jsonLexer; diff --git a/core/src/main/java/com/questdb/griffin/SqlCompiler.java b/core/src/main/java/com/questdb/griffin/SqlCompiler.java index d50c3f2793f54c47c3dc5f5eb91e1727ce2e13ef..28646b0e546215e3b979b084bc68979f150b3004 100644 --- a/core/src/main/java/com/questdb/griffin/SqlCompiler.java +++ b/core/src/main/java/com/questdb/griffin/SqlCompiler.java @@ -78,7 +78,7 @@ public class SqlCompiler implements Closeable { this.queryModelPool = new ObjectPool<>(QueryModel.FACTORY, configuration.getSqlModelPoolCapacity()); this.characterStore = new CharacterStore( configuration.getSqlCharacterStoreCapacity(), - configuration.getSqlCharacterStorePoolCapacity() + configuration.getSqlCharacterStoreSequencePoolCapacity() ); this.lexer = new GenericLexer(configuration.getSqlLexerPoolCapacity()); final FunctionParser functionParser = new FunctionParser(configuration, ServiceLoader.load(FunctionFactory.class)); @@ -110,8 +110,8 @@ public class SqlCompiler implements Closeable { ); this.sqlCache = new AssociativeCache<>( - configuration.getSqlCacheBlockSize(), - configuration.getSqlCacheBlockCount() + configuration.getSqlCacheBlocks(), + configuration.getSqlCacheRows() ); } diff --git a/core/src/main/java/com/questdb/network/DefaultIODispatcherConfiguration.java b/core/src/main/java/com/questdb/network/DefaultIODispatcherConfiguration.java index bee8ac179608b4b36cf8c09249e039eafd2361a6..90ecb32be292790d68e5f7f2c21112f8ac9b9164 100644 --- a/core/src/main/java/com/questdb/network/DefaultIODispatcherConfiguration.java +++ b/core/src/main/java/com/questdb/network/DefaultIODispatcherConfiguration.java @@ -34,8 +34,8 @@ public class DefaultIODispatcherConfiguration implements IODispatcherConfigurati } @Override - public CharSequence getBindIPv4Address() { - return "0.0.0.0"; + public int getBindIPv4Address() { + return 0; } @Override @@ -60,7 +60,7 @@ public class DefaultIODispatcherConfiguration implements IODispatcherConfigurati @Override public long getIdleConnectionTimeout() { - return 10000000000000000L; + return 5 * 60 * 1000L; } @Override diff --git a/core/src/main/java/com/questdb/network/IODispatcherConfiguration.java b/core/src/main/java/com/questdb/network/IODispatcherConfiguration.java index b4528e9ffb3c6cb770c4649d2557c267058b91c6..ab60446493f2fb48dcd6cad7c282dcff063871fc 100644 --- a/core/src/main/java/com/questdb/network/IODispatcherConfiguration.java +++ b/core/src/main/java/com/questdb/network/IODispatcherConfiguration.java @@ -31,7 +31,7 @@ public interface IODispatcherConfiguration { int getActiveConnectionLimit(); - CharSequence getBindIPv4Address(); + int getBindIPv4Address(); int getBindPort(); diff --git a/core/src/test/java/com/questdb/PropServerConfigurationTest.java b/core/src/test/java/com/questdb/PropServerConfigurationTest.java new file mode 100644 index 0000000000000000000000000000000000000000..2cf25dc9fd878546db71a2a6c61ae6bbe504d489 --- /dev/null +++ b/core/src/test/java/com/questdb/PropServerConfigurationTest.java @@ -0,0 +1,293 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (C) 2014-2019 Appsicle + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + ******************************************************************************/ + +package com.questdb; + +import com.questdb.network.EpollFacadeImpl; +import com.questdb.network.IOOperation; +import com.questdb.network.NetworkFacadeImpl; +import com.questdb.network.SelectFacadeImpl; +import com.questdb.std.FilesFacadeImpl; +import com.questdb.std.microtime.MicrosecondClockImpl; +import com.questdb.std.time.MillisecondClockImpl; +import com.questdb.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class PropServerConfigurationTest { + + @Test + public void testAllDefaults() throws ServerConfigurationException { + Properties properties = new Properties(); + PropServerConfiguration configuration = new PropServerConfiguration("root", properties); + Assert.assertEquals(16, configuration.getHttpServerConfiguration().getConnectionPoolInitialCapacity()); + Assert.assertEquals(128, configuration.getHttpServerConfiguration().getConnectionStringPoolCapacity()); + Assert.assertEquals(512, configuration.getHttpServerConfiguration().getMultipartHeaderBufferSize()); + Assert.assertEquals(10_000, configuration.getHttpServerConfiguration().getMultipartIdleSpinCount()); + Assert.assertEquals(1048576, configuration.getHttpServerConfiguration().getRecvBufferSize()); + Assert.assertEquals(1024, configuration.getHttpServerConfiguration().getRequestHeaderBufferSize()); + Assert.assertEquals(1048576, configuration.getHttpServerConfiguration().getResponseHeaderBufferSize()); + Assert.assertEquals(2, configuration.getHttpServerConfiguration().getWorkerCount()); + Assert.assertEquals(2097152, configuration.getHttpServerConfiguration().getSendBufferSize()); + Assert.assertEquals("index.html", configuration.getHttpServerConfiguration().getStaticContentProcessorConfiguration().getIndexFileName()); + Assert.assertEquals("public", configuration.getHttpServerConfiguration().getStaticContentProcessorConfiguration().getPublicDirectory()); + Assert.assertTrue(configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().abortBrokenUploads()); + + Assert.assertEquals(256, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getActiveConnectionLimit()); + Assert.assertEquals(1024, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getEventCapacity()); + Assert.assertEquals(1024, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getIOQueueCapacity()); + Assert.assertEquals(300000, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getIdleConnectionTimeout()); + Assert.assertEquals(1024, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getInterestQueueCapacity()); + Assert.assertEquals(IOOperation.READ, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getInitialBias()); + Assert.assertEquals(256, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getListenBacklog()); + Assert.assertEquals(2097152, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getSndBufSize()); + Assert.assertEquals(2097152, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getRcvBufSize()); + Assert.assertEquals("/text_loader.json", configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getAdapterSetConfigurationFileName()); + Assert.assertEquals(16, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getDateAdapterPoolCapacity()); + Assert.assertEquals(16384, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getJsonCacheLimit()); + Assert.assertEquals(8192, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getJsonCacheSize()); + Assert.assertEquals(0.1222d, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getMaxRequiredDelimiterStdDev(), 0.000000001); + Assert.assertEquals(128, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getMetadataStringPoolCapacity()); + Assert.assertEquals(4096, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getRollBufferLimit()); + Assert.assertEquals(1024, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getRollBufferSize()); + Assert.assertEquals(1000, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getTextAnalysisMaxLines()); + Assert.assertEquals(64, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getTextLexerStringPoolCapacity()); + Assert.assertEquals(64, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getTimestampAdapterPoolCapacity()); + Assert.assertEquals(4096, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getUtf8SinkSize()); + Assert.assertEquals(0, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getBindIPv4Address()); + Assert.assertEquals(9000, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getBindPort()); + + Assert.assertEquals(5, configuration.getCairoConfiguration().getCreateAsSelectRetryCount()); + Assert.assertEquals("fast", configuration.getCairoConfiguration().getDefaultMapType()); + Assert.assertFalse(configuration.getCairoConfiguration().getDefaultSymbolCacheFlag()); + Assert.assertEquals(256, configuration.getCairoConfiguration().getDefaultSymbolCapacity()); + Assert.assertEquals(30, configuration.getCairoConfiguration().getFileOperationRetryCount()); + Assert.assertEquals(100, configuration.getCairoConfiguration().getIdleCheckInterval()); + Assert.assertEquals(-10_000, configuration.getCairoConfiguration().getInactiveReaderTTL()); + Assert.assertEquals(-10_000, configuration.getCairoConfiguration().getInactiveWriterTTL()); + Assert.assertEquals(256, configuration.getCairoConfiguration().getIndexValueBlockSize()); + Assert.assertEquals(30, configuration.getCairoConfiguration().getMaxSwapFileCount()); + Assert.assertEquals(509, configuration.getCairoConfiguration().getMkDirMode()); + + Assert.assertEquals(100000, configuration.getCairoConfiguration().getParallelIndexThreshold()); + Assert.assertEquals(5, configuration.getCairoConfiguration().getReaderPoolMaxSegments()); + Assert.assertEquals(1_000_000, configuration.getCairoConfiguration().getSpinLockTimeoutUs()); + Assert.assertEquals(16, configuration.getCairoConfiguration().getSqlCacheRows()); + Assert.assertEquals(4, configuration.getCairoConfiguration().getSqlCacheBlocks()); + Assert.assertEquals(1024, configuration.getCairoConfiguration().getSqlCharacterStoreCapacity()); + Assert.assertEquals(64, configuration.getCairoConfiguration().getSqlCharacterStoreSequencePoolCapacity()); + Assert.assertEquals(4096, configuration.getCairoConfiguration().getSqlColumnPoolCapacity()); + Assert.assertEquals(0.7, configuration.getCairoConfiguration().getSqlCompactMapLoadFactor(), 0.000001); + Assert.assertEquals(8192, configuration.getCairoConfiguration().getSqlExpressionPoolCapacity()); + Assert.assertEquals(0.5, configuration.getCairoConfiguration().getSqlFastMapLoadFactor(), 0.0000001); + Assert.assertEquals(64, configuration.getCairoConfiguration().getSqlJoinContextPoolCapacity()); + Assert.assertEquals(2048, configuration.getCairoConfiguration().getSqlLexerPoolCapacity()); + Assert.assertEquals(2048, configuration.getCairoConfiguration().getSqlMapKeyCapacity()); + Assert.assertEquals(4 * 1024 * 1024, configuration.getCairoConfiguration().getSqlMapPageSize()); + Assert.assertEquals(1024, configuration.getCairoConfiguration().getSqlModelPoolCapacity()); + Assert.assertEquals(4 * 1024 * 1024, configuration.getCairoConfiguration().getSqlSortKeyPageSize()); + Assert.assertEquals(1024 * 1024, configuration.getCairoConfiguration().getSqlSortLightValuePageSize()); + Assert.assertEquals(16 * 1024 * 1024, configuration.getCairoConfiguration().getSqlHashJoinValuePageSize()); + Assert.assertEquals(4 * 1024 * 1024, configuration.getCairoConfiguration().getSqlTreePageSize()); + Assert.assertEquals(1024 * 1024, configuration.getCairoConfiguration().getSqlHashJoinLightValuePageSize()); + Assert.assertEquals(16 * 1024 * 1024, configuration.getCairoConfiguration().getSqlSortValuePageSize()); + Assert.assertEquals(10000, configuration.getCairoConfiguration().getWorkStealTimeoutNanos()); + Assert.assertTrue(configuration.getCairoConfiguration().isParallelIndexingEnabled()); + Assert.assertEquals(16 * 1024, configuration.getCairoConfiguration().getSqlJoinMetadataPageSize()); + + Assert.assertEquals(0, configuration.getLineUdpReceiverConfiguration().getBindIPv4Address()); + Assert.assertEquals(9009, configuration.getLineUdpReceiverConfiguration().getPort()); + Assert.assertEquals(-402587133, configuration.getLineUdpReceiverConfiguration().getGroupIPv4Address()); + + Assert.assertEquals(10000, configuration.getLineUdpReceiverConfiguration().getCommitRate()); + + Assert.assertEquals(1024 * 1024, configuration.getLineUdpReceiverConfiguration().getMsgBufferSize()); + Assert.assertEquals(10000, configuration.getLineUdpReceiverConfiguration().getMsgCount()); + Assert.assertEquals(2048, configuration.getLineUdpReceiverConfiguration().getReceiveBufferSize()); + + // statics + Assert.assertSame(FilesFacadeImpl.INSTANCE, configuration.getHttpServerConfiguration().getStaticContentProcessorConfiguration().getFilesFacade()); + Assert.assertSame(MillisecondClockImpl.INSTANCE, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getClock()); + Assert.assertSame(MillisecondClockImpl.INSTANCE, configuration.getHttpServerConfiguration().getClock()); + Assert.assertSame(NetworkFacadeImpl.INSTANCE, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getNetworkFacade()); + Assert.assertSame(EpollFacadeImpl.INSTANCE, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getEpollFacade()); + Assert.assertSame(SelectFacadeImpl.INSTANCE, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getSelectFacade()); + Assert.assertSame(FilesFacadeImpl.INSTANCE, configuration.getCairoConfiguration().getFilesFacade()); + Assert.assertSame(MillisecondClockImpl.INSTANCE, configuration.getCairoConfiguration().getMillisecondClock()); + Assert.assertSame(MicrosecondClockImpl.INSTANCE, configuration.getCairoConfiguration().getMicrosecondClock()); + Assert.assertSame(NetworkFacadeImpl.INSTANCE, configuration.getLineUdpReceiverConfiguration().getNetworkFacade()); + TestUtils.assertEquals("root", configuration.getCairoConfiguration().getRoot()); + + // assert mime types + TestUtils.assertEquals("application/json", configuration.getHttpServerConfiguration().getStaticContentProcessorConfiguration().getMimeTypesCache().get("json")); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidBindToAddress() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("http.bind.to", "10.5.6:8990"); + new PropServerConfiguration("root", properties); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidBindToMissingColon() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("http.bind.to", "10.5.6.1"); + new PropServerConfiguration("root", properties); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidBindToPort() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("http.bind.to", "10.5.6.1:"); + new PropServerConfiguration("root", properties); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidDouble() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("http.text.max.required.delimiter.stddev", "abc"); + new PropServerConfiguration("root", properties); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidIPv4Address() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("line.udp.join", "12a.990.00"); + new PropServerConfiguration("root", properties); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidInt() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("http.connection.string.pool.capacity", "1234a"); + new PropServerConfiguration("root", properties); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidIntSize() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("http.request.header.buffer.size", "22g"); + new PropServerConfiguration("root", properties); + } + + @Test(expected = ServerConfigurationException.class) + public void testInvalidLong() throws ServerConfigurationException { + Properties properties = new Properties(); + properties.setProperty("cairo.idle.check.interval", "1234a"); + new PropServerConfiguration("root", properties); + } + + @Test + public void testSetAllFromFile() throws IOException, ServerConfigurationException { + try (InputStream is = PropServerConfigurationTest.class.getResourceAsStream("/server.conf")) { + Properties properties = new Properties(); + properties.load(is); + PropServerConfiguration configuration = new PropServerConfiguration("data", properties); + Assert.assertEquals(64, configuration.getHttpServerConfiguration().getConnectionPoolInitialCapacity()); + Assert.assertEquals(512, configuration.getHttpServerConfiguration().getConnectionStringPoolCapacity()); + Assert.assertEquals(256, configuration.getHttpServerConfiguration().getMultipartHeaderBufferSize()); + Assert.assertEquals(100_000, configuration.getHttpServerConfiguration().getMultipartIdleSpinCount()); + Assert.assertEquals(4096, configuration.getHttpServerConfiguration().getRecvBufferSize()); + Assert.assertEquals(2048, configuration.getHttpServerConfiguration().getRequestHeaderBufferSize()); + Assert.assertEquals(9012, configuration.getHttpServerConfiguration().getResponseHeaderBufferSize()); + Assert.assertEquals(6, configuration.getHttpServerConfiguration().getWorkerCount()); + Assert.assertEquals(128, configuration.getHttpServerConfiguration().getSendBufferSize()); + Assert.assertEquals("index2.html", configuration.getHttpServerConfiguration().getStaticContentProcessorConfiguration().getIndexFileName()); + Assert.assertEquals("public_ok", configuration.getHttpServerConfiguration().getStaticContentProcessorConfiguration().getPublicDirectory()); + Assert.assertFalse(configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().abortBrokenUploads()); + Assert.assertEquals(64, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getActiveConnectionLimit()); + Assert.assertEquals(2048, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getEventCapacity()); + Assert.assertEquals(64, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getIOQueueCapacity()); + Assert.assertEquals(7000000, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getIdleConnectionTimeout()); + Assert.assertEquals(512, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getInterestQueueCapacity()); + Assert.assertEquals(IOOperation.READ, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getInitialBias()); + Assert.assertEquals(64, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getListenBacklog()); + Assert.assertEquals(4194304, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getSndBufSize()); + Assert.assertEquals(8388608, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getRcvBufSize()); + Assert.assertEquals("/loader.json", configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getAdapterSetConfigurationFileName()); + Assert.assertEquals(32, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getDateAdapterPoolCapacity()); + Assert.assertEquals(65536, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getJsonCacheLimit()); + Assert.assertEquals(8388608, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getJsonCacheSize()); + Assert.assertEquals(0.3d, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getMaxRequiredDelimiterStdDev(), 0.000000001); + Assert.assertEquals(512, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getMetadataStringPoolCapacity()); + Assert.assertEquals(6144, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getRollBufferLimit()); + Assert.assertEquals(3072, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getRollBufferSize()); + Assert.assertEquals(400, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getTextAnalysisMaxLines()); + Assert.assertEquals(128, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getTextLexerStringPoolCapacity()); + Assert.assertEquals(512, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getTimestampAdapterPoolCapacity()); + Assert.assertEquals(8192, configuration.getHttpServerConfiguration().getTextImportProcessorConfiguration().getTextConfiguration().getUtf8SinkSize()); + Assert.assertEquals(168101918, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getBindIPv4Address()); + Assert.assertEquals(9900, configuration.getHttpServerConfiguration().getDispatcherConfiguration().getBindPort()); + + Assert.assertEquals(12, configuration.getCairoConfiguration().getCreateAsSelectRetryCount()); + Assert.assertEquals("compact", configuration.getCairoConfiguration().getDefaultMapType()); + Assert.assertTrue(configuration.getCairoConfiguration().getDefaultSymbolCacheFlag()); + Assert.assertEquals(512, configuration.getCairoConfiguration().getDefaultSymbolCapacity()); + Assert.assertEquals(10, configuration.getCairoConfiguration().getFileOperationRetryCount()); + Assert.assertEquals(20_000, configuration.getCairoConfiguration().getIdleCheckInterval()); + Assert.assertEquals(600_000, configuration.getCairoConfiguration().getInactiveReaderTTL()); + Assert.assertEquals(400_000, configuration.getCairoConfiguration().getInactiveWriterTTL()); + Assert.assertEquals(1024, configuration.getCairoConfiguration().getIndexValueBlockSize()); + Assert.assertEquals(23, configuration.getCairoConfiguration().getMaxSwapFileCount()); + Assert.assertEquals(580, configuration.getCairoConfiguration().getMkDirMode()); + Assert.assertEquals(1000000, configuration.getCairoConfiguration().getParallelIndexThreshold()); + Assert.assertEquals(10, configuration.getCairoConfiguration().getReaderPoolMaxSegments()); + Assert.assertEquals(5_000_000, configuration.getCairoConfiguration().getSpinLockTimeoutUs()); + Assert.assertEquals(32, configuration.getCairoConfiguration().getSqlCacheRows()); + Assert.assertEquals(16, configuration.getCairoConfiguration().getSqlCacheBlocks()); + Assert.assertEquals(2048, configuration.getCairoConfiguration().getSqlCharacterStoreCapacity()); + Assert.assertEquals(128, configuration.getCairoConfiguration().getSqlCharacterStoreSequencePoolCapacity()); + Assert.assertEquals(2048, configuration.getCairoConfiguration().getSqlColumnPoolCapacity()); + Assert.assertEquals(0.8, configuration.getCairoConfiguration().getSqlCompactMapLoadFactor(), 0.000001); + Assert.assertEquals(1024, configuration.getCairoConfiguration().getSqlExpressionPoolCapacity()); + Assert.assertEquals(0.3, configuration.getCairoConfiguration().getSqlFastMapLoadFactor(), 0.0000001); + Assert.assertEquals(32, configuration.getCairoConfiguration().getSqlJoinContextPoolCapacity()); + Assert.assertEquals(1024, configuration.getCairoConfiguration().getSqlLexerPoolCapacity()); + Assert.assertEquals(1024, configuration.getCairoConfiguration().getSqlMapKeyCapacity()); + Assert.assertEquals(6 * 1024 * 1024, configuration.getCairoConfiguration().getSqlMapPageSize()); + Assert.assertEquals(256, configuration.getCairoConfiguration().getSqlModelPoolCapacity()); + Assert.assertEquals(10 * 1024 * 1024, configuration.getCairoConfiguration().getSqlSortKeyPageSize()); + Assert.assertEquals(3 * 1024 * 1024, configuration.getCairoConfiguration().getSqlSortLightValuePageSize()); + Assert.assertEquals(8 * 1024 * 1024, configuration.getCairoConfiguration().getSqlHashJoinValuePageSize()); + Assert.assertEquals(16 * 1024 * 1024, configuration.getCairoConfiguration().getSqlTreePageSize()); + Assert.assertEquals(2 * 1024 * 1024, configuration.getCairoConfiguration().getSqlHashJoinLightValuePageSize()); + Assert.assertEquals(4 * 1024 * 1024, configuration.getCairoConfiguration().getSqlSortValuePageSize()); + Assert.assertEquals(1000000, configuration.getCairoConfiguration().getWorkStealTimeoutNanos()); + Assert.assertFalse(configuration.getCairoConfiguration().isParallelIndexingEnabled()); + Assert.assertEquals(8 * 1024, configuration.getCairoConfiguration().getSqlJoinMetadataPageSize()); + + Assert.assertEquals(167903521, configuration.getLineUdpReceiverConfiguration().getBindIPv4Address()); + Assert.assertEquals(9915, configuration.getLineUdpReceiverConfiguration().getPort()); + Assert.assertEquals(-536805119, configuration.getLineUdpReceiverConfiguration().getGroupIPv4Address()); + Assert.assertEquals(100_000, configuration.getLineUdpReceiverConfiguration().getCommitRate()); + Assert.assertEquals(4 * 1024 * 1024, configuration.getLineUdpReceiverConfiguration().getMsgBufferSize()); + Assert.assertEquals(4000, configuration.getLineUdpReceiverConfiguration().getMsgCount()); + Assert.assertEquals(512, configuration.getLineUdpReceiverConfiguration().getReceiveBufferSize()); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/questdb/cutlass/http/IODispatcherTest.java b/core/src/test/java/com/questdb/cutlass/http/IODispatcherTest.java index 9eabb495ae5bfe2c475fb02e2631d48bc922a7f7..2d0529b7e75f6a7e31ba44301812e9590c9df2d5 100644 --- a/core/src/test/java/com/questdb/cutlass/http/IODispatcherTest.java +++ b/core/src/test/java/com/questdb/cutlass/http/IODispatcherTest.java @@ -765,141 +765,6 @@ public class IODispatcherTest { }); } - @Test - public void testJsonQuerySyntaxError() throws Exception { - TestUtils.assertMemoryLeak(() -> { - final String baseDir = System.getProperty("java.io.tmpdir"); - final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir); - - try ( - CairoEngine engine = new Engine(new DefaultCairoConfiguration(baseDir)); - HttpServer httpServer = new HttpServer(httpConfiguration) - ) { - httpServer.bind(new HttpRequestProcessorFactory() { - @Override - public String getUrl() { - return HttpServerConfiguration.DEFAULT_PROCESSOR_URL; - } - - @Override - public HttpRequestProcessor newInstance() { - return new StaticContentProcessor(httpConfiguration.getStaticContentProcessorConfiguration()); - } - }); - - httpServer.bind(new HttpRequestProcessorFactory() { - @Override - public String getUrl() { - return "/query"; - } - - @Override - public HttpRequestProcessor newInstance() { - return new JsonQueryProcessor(engine); - } - }); - - httpServer.start(); - - // create table with all column types - CairoTestUtils.createTestTable( - engine.getConfiguration(), - 20, - new Rnd(), - new TestRecord.ArrayBinarySequence()); - - // send multipart request to server - final String request = "GET /query?query=x%20where2%20i%20%3D%20(%27EHNRX%27) HTTP/1.1\r\n" + - "Host: localhost:9001\r\n" + - "Connection: keep-alive\r\n" + - "Cache-Control: max-age=0\r\n" + - "Upgrade-Insecure-Requests: 1\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + - "Accept-Encoding: gzip, deflate, br\r\n" + - "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + - "\r\n"; - - byte[] expectedResponse = ("HTTP/1.1 200 OK\r\n" + - "Server: questDB/1.0\r\n" + - "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + - "Transfer-Encoding: chunked\r\n" + - "Content-Type: application/json; charset=utf-8\r\n" + - "Keep-Alive: timeout=5, max=10000\r\n" + - "\r\n" + - "224\r\n" + - "{\"query\":\"x where i = ('EHNRX')\",\"columns\":[{\"name\":\"a\",\"type\":\"BYTE\"},{\"name\":\"b\",\"type\":\"SHORT\"},{\"name\":\"c\",\"type\":\"INT\"},{\"name\":\"d\",\"type\":\"LONG\"},{\"name\":\"e\",\"type\":\"DATE\"},{\"name\":\"f\",\"type\":\"TIMESTAMP\"},{\"name\":\"g\",\"type\":\"FLOAT\"},{\"name\":\"h\",\"type\":\"DOUBLE\"},{\"name\":\"i\",\"type\":\"STRING\"},{\"name\":\"j\",\"type\":\"SYMBOL\"},{\"name\":\"k\",\"type\":\"BOOLEAN\"},{\"name\":\"l\",\"type\":\"BINARY\"}],\"dataset\":[[80,24814,-727724771,8920866532787660373,\"-169665660-01-09T01:58:28.119Z\",\"-51129-02-11T06:38:29.397464Z\",null,null,\"EHNRX\",\"ZSX\",false,[]]],\"count\":1}\r\n" + - "0\r\n" + - "\r\n").getBytes(); - - sendAndReceive( - NetworkFacadeImpl.INSTANCE, - request, - expectedResponse, - 10, - 0 - ); - - httpServer.halt(); - } - }); - } - - @Test - @Ignore - public void testUpload() throws Exception { - TestUtils.assertMemoryLeak(() -> { - final String baseDir = System.getProperty("java.io.tmpdir"); -// final String baseDir = "/home/vlad/dev/123"; - final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir); - - - try (CairoEngine engine = new Engine(new DefaultCairoConfiguration(baseDir)); - HttpServer httpServer = new HttpServer(httpConfiguration)) { - httpServer.bind(new HttpRequestProcessorFactory() { - @Override - public String getUrl() { - return HttpServerConfiguration.DEFAULT_PROCESSOR_URL; - } - - @Override - public HttpRequestProcessor newInstance() { - return new StaticContentProcessor(httpConfiguration.getStaticContentProcessorConfiguration()); - } - }); - - httpServer.bind(new HttpRequestProcessorFactory() { - @Override - public String getUrl() { - return "/upload"; - } - - @Override - public HttpRequestProcessor newInstance() { - return new TextImportProcessor(httpConfiguration.getTextImportProcessorConfiguration(), engine); - } - }); - - httpServer.bind(new HttpRequestProcessorFactory() { - @Override - public String getUrl() { - return "/query"; - } - - @Override - public HttpRequestProcessor newInstance() { - return new JsonQueryProcessor(engine); - } - }); - - - httpServer.start(); - - Thread.sleep(2000000); - } - }); - } - @Test public void testJsonQueryAndDisconnectWithoutWaitingForResult() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -1045,6 +910,86 @@ public class IODispatcherTest { }); } + @Test + public void testJsonQuerySyntaxError() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String baseDir = System.getProperty("java.io.tmpdir"); + final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir); + + try ( + CairoEngine engine = new Engine(new DefaultCairoConfiguration(baseDir)); + HttpServer httpServer = new HttpServer(httpConfiguration) + ) { + httpServer.bind(new HttpRequestProcessorFactory() { + @Override + public String getUrl() { + return HttpServerConfiguration.DEFAULT_PROCESSOR_URL; + } + + @Override + public HttpRequestProcessor newInstance() { + return new StaticContentProcessor(httpConfiguration.getStaticContentProcessorConfiguration()); + } + }); + + httpServer.bind(new HttpRequestProcessorFactory() { + @Override + public String getUrl() { + return "/query"; + } + + @Override + public HttpRequestProcessor newInstance() { + return new JsonQueryProcessor(engine); + } + }); + + httpServer.start(); + + // create table with all column types + CairoTestUtils.createTestTable( + engine.getConfiguration(), + 20, + new Rnd(), + new TestRecord.ArrayBinarySequence()); + + // send multipart request to server + final String request = "GET /query?query=x%20where2%20i%20%3D%20(%27EHNRX%27) HTTP/1.1\r\n" + + "Host: localhost:9001\r\n" + + "Connection: keep-alive\r\n" + + "Cache-Control: max-age=0\r\n" + + "Upgrade-Insecure-Requests: 1\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + + "Accept-Encoding: gzip, deflate, br\r\n" + + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + + "\r\n"; + + byte[] expectedResponse = ("HTTP/1.1 400 Bad request\r\n" + + "Server: questDB/1.0\r\n" + + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Type: application/json; charset=utf-8\r\n" + + "Keep-Alive: timeout=5, max=10000\r\n" + + "\r\n" + + "4d\r\n" + + "{\"query\":\"x where2 i = ('EHNRX')\",\"error\":\"unexpected token: i\",\"position\":9}\r\n" + + "0\r\n" + + "\r\n").getBytes(); + + sendAndReceive( + NetworkFacadeImpl.INSTANCE, + request, + expectedResponse, + 10, + 0 + ); + + httpServer.halt(); + } + }); + } + @Test public void testMaxConnections() throws Exception { @@ -2149,37 +2094,59 @@ public class IODispatcherTest { }); } - private static class CairoHttpServer { - private final CairoEngine engine; - private final HttpServer httpServer; + @Test + @Ignore + public void testUpload() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String baseDir = System.getProperty("java.io.tmpdir"); +// final String baseDir = "/home/vlad/dev/123"; + final DefaultHttpServerConfiguration httpConfiguration = createHttpServerConfiguration(baseDir); - public CairoHttpServer(CharSequence cairoBaseDir, HttpServerConfiguration configuration) { - this.engine = new Engine(new DefaultCairoConfiguration(cairoBaseDir)); - this.httpServer = new HttpServer(configuration); - httpServer.bind(new HttpRequestProcessorFactory() { - @Override - public String getUrl() { - return HttpServerConfiguration.DEFAULT_PROCESSOR_URL; - } - @Override - public HttpRequestProcessor newInstance() { - return new StaticContentProcessor(configuration.getStaticContentProcessorConfiguration()); - } - }); + try (CairoEngine engine = new Engine(new DefaultCairoConfiguration(baseDir)); + HttpServer httpServer = new HttpServer(httpConfiguration)) { + httpServer.bind(new HttpRequestProcessorFactory() { + @Override + public String getUrl() { + return HttpServerConfiguration.DEFAULT_PROCESSOR_URL; + } - httpServer.bind(new HttpRequestProcessorFactory() { - @Override - public String getUrl() { - return "/query"; - } + @Override + public HttpRequestProcessor newInstance() { + return new StaticContentProcessor(httpConfiguration.getStaticContentProcessorConfiguration()); + } + }); - @Override - public HttpRequestProcessor newInstance() { - return new JsonQueryProcessor(engine); - } - }); - } + httpServer.bind(new HttpRequestProcessorFactory() { + @Override + public String getUrl() { + return "/upload"; + } + + @Override + public HttpRequestProcessor newInstance() { + return new TextImportProcessor(httpConfiguration.getTextImportProcessorConfiguration(), engine); + } + }); + + httpServer.bind(new HttpRequestProcessorFactory() { + @Override + public String getUrl() { + return "/query"; + } + + @Override + public HttpRequestProcessor newInstance() { + return new JsonQueryProcessor(engine); + } + }); + + + httpServer.start(); + + Thread.sleep(2000000); + } + }); } private static void assertDownloadResponse(long fd, Rnd rnd, long buffer, int len, int nonRepeatedContentLength, String expectedResponseHeader, long expectedResponseLen) { @@ -2320,6 +2287,7 @@ public class IODispatcherTest { while (received < expectedToReceive) { int n = nf.recv(fd, ptr + received, len - received); if (n > 0) { +// dump(ptr + received, n); // compare bytes for (int i = 0; i < n; i++) { if (expectedResponse[received + i] != Unsafe.getUnsafe().getByte(ptr + received + i)) { @@ -2364,6 +2332,39 @@ public class IODispatcherTest { Unsafe.free(buf, bufLen); } + private static class CairoHttpServer { + private final CairoEngine engine; + private final HttpServer httpServer; + + public CairoHttpServer(CharSequence cairoBaseDir, HttpServerConfiguration configuration) { + this.engine = new Engine(new DefaultCairoConfiguration(cairoBaseDir)); + this.httpServer = new HttpServer(configuration); + httpServer.bind(new HttpRequestProcessorFactory() { + @Override + public String getUrl() { + return HttpServerConfiguration.DEFAULT_PROCESSOR_URL; + } + + @Override + public HttpRequestProcessor newInstance() { + return new StaticContentProcessor(configuration.getStaticContentProcessorConfiguration()); + } + }); + + httpServer.bind(new HttpRequestProcessorFactory() { + @Override + public String getUrl() { + return "/query"; + } + + @Override + public HttpRequestProcessor newInstance() { + return new JsonQueryProcessor(engine); + } + }); + } + } + private static class HelloContext implements IOContext { private final long fd; private final long buffer = Unsafe.malloc(1024); diff --git a/core/src/test/java/com/questdb/cutlass/line/CairoLineProtoParserTest.java b/core/src/test/java/com/questdb/cutlass/line/CairoLineProtoParserTest.java index 59ac3987fed3077c9f6de341872bf75aaf082168..70108c77890c6fdce620ef29f93b4dcd04164d25 100644 --- a/core/src/test/java/com/questdb/cutlass/line/CairoLineProtoParserTest.java +++ b/core/src/test/java/com/questdb/cutlass/line/CairoLineProtoParserTest.java @@ -24,7 +24,7 @@ package com.questdb.cutlass.line; import com.questdb.cairo.*; -import com.questdb.cairo.pool.WriterPool; +import com.questdb.cairo.sql.CairoEngine; import com.questdb.std.*; import com.questdb.std.microtime.DateFormatUtils; import com.questdb.std.microtime.MicrosecondClock; @@ -506,8 +506,8 @@ public class CairoLineProtoParserTest extends AbstractCairoTest { private void assertThat(String expected, String lines, CharSequence tableName, CairoConfiguration configuration) throws Exception { TestUtils.assertMemoryLeak(() -> { - try (WriterPool pool = new WriterPool(configuration, null)) { - try (CairoLineProtoParser parser = new CairoLineProtoParser(configuration, pool)) { + try (CairoEngine engine = new Engine(configuration, null)) { + try (CairoLineProtoParser parser = new CairoLineProtoParser(engine)) { byte[] bytes = lines.getBytes(StandardCharsets.UTF_8); int len = bytes.length; long mem = Unsafe.malloc(len); diff --git a/core/src/test/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiverTest.java b/core/src/test/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiverTest.java index 2fd4698f1565be41fb12a63c37998c3768ae27e5..e5e6088fe94e1ca78ff7044ac4a85f215dbd0621 100644 --- a/core/src/test/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiverTest.java +++ b/core/src/test/java/com/questdb/cutlass/line/udp/LinuxLineProtoReceiverTest.java @@ -24,7 +24,7 @@ package com.questdb.cutlass.line.udp; import com.questdb.cairo.*; -import com.questdb.cairo.pool.WriterPool; +import com.questdb.cairo.sql.CairoEngine; import com.questdb.mp.Job; import com.questdb.mp.SOCountDownLatch; import com.questdb.mp.Worker; @@ -73,7 +73,7 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { @Test public void testGenericSimpleReceive() throws Exception { - assertReceive(new TestReceiverConfiguration(), GENERIC_FACTORY); + assertReceive(new TestLineUdpReceiverConfiguration(), GENERIC_FACTORY); } @Test @@ -121,7 +121,7 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { if (Os.type != Os.LINUX) { return; } - assertReceive(new TestReceiverConfiguration(), LINUX_FACTORY); + assertReceive(new TestLineUdpReceiverConfiguration(), LINUX_FACTORY); } private void assertCannotBindSocket(ReceiverFactory factory) throws Exception { @@ -132,7 +132,7 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { return false; } }; - ReceiverConfiguration receiverCfg = new TestReceiverConfiguration() { + LineUdpReceiverConfiguration receiverCfg = new TestLineUdpReceiverConfiguration() { @Override public NetworkFacade getNetworkFacade() { return nf; @@ -146,12 +146,12 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { TestUtils.assertMemoryLeak(() -> { NetworkFacade nf = new NetworkFacadeImpl() { @Override - public boolean join(long fd, CharSequence bindIPv4Address, CharSequence groupIPv4Address) { + public boolean join(long fd, int bindIPv4Address, int groupIPv4Address) { return false; } }; - ReceiverConfiguration receiverCfg = new TestReceiverConfiguration() { + LineUdpReceiverConfiguration receiverCfg = new TestLineUdpReceiverConfiguration() { @Override public NetworkFacade getNetworkFacade() { return nf; @@ -170,7 +170,7 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { return -1; } }; - ReceiverConfiguration receiverCfg = new TestReceiverConfiguration() { + LineUdpReceiverConfiguration receiverCfg = new TestLineUdpReceiverConfiguration() { @Override public NetworkFacade getNetworkFacade() { return nf; @@ -188,25 +188,24 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { } }; - ReceiverConfiguration configuration = new TestReceiverConfiguration() { + LineUdpReceiverConfiguration configuration = new TestLineUdpReceiverConfiguration() { @Override - public int getReceiveBufferSize() { - return 2048; + public NetworkFacade getNetworkFacade() { + return nf; } @Override - public NetworkFacade getNetworkFacade() { - return nf; + public int getReceiveBufferSize() { + return 2048; } }; assertReceive(configuration, factory); } - private void assertConstructorFail(ReceiverConfiguration receiverCfg, ReceiverFactory factory) { - CairoConfiguration cairoCfg = new DefaultCairoConfiguration(root); - try (WriterPool pool = new WriterPool(cairoCfg, null)) { + private void assertConstructorFail(LineUdpReceiverConfiguration receiverCfg, ReceiverFactory factory) { + try (CairoEngine engine = new Engine(new DefaultCairoConfiguration(root), null)) { try { - factory.createReceiver(receiverCfg, cairoCfg, pool); + factory.createReceiver(receiverCfg, engine); Assert.fail(); } catch (CairoException ignore) { } @@ -214,7 +213,7 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { } private void assertFrequentCommit(ReceiverFactory factory) throws Exception { - ReceiverConfiguration configuration = new TestReceiverConfiguration() { + LineUdpReceiverConfiguration configuration = new TestLineUdpReceiverConfiguration() { @Override public int getCommitRate() { return 0; @@ -223,7 +222,7 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { assertReceive(configuration, factory); } - private void assertReceive(ReceiverConfiguration receiverCfg, ReceiverFactory factory) throws Exception { + private void assertReceive(LineUdpReceiverConfiguration receiverCfg, ReceiverFactory factory) throws Exception { TestUtils.assertMemoryLeak(() -> { final String expected = "colour\tshape\tsize\ttimestamp\n" + "blue\tsquare\t3.400000000000\t1970-01-01T00:01:40.000000Z\n" + @@ -237,10 +236,9 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { "blue\tsquare\t3.400000000000\t1970-01-01T00:01:40.000000Z\n" + "blue\tsquare\t3.400000000000\t1970-01-01T00:01:40.000000Z\n"; - CairoConfiguration cairoCfg = new DefaultCairoConfiguration(root); - try (WriterPool pool = new WriterPool(cairoCfg, null)) { + try (CairoEngine engine = new Engine(new DefaultCairoConfiguration(root), null)) { - Job receiver = factory.createReceiver(receiverCfg, cairoCfg, pool); + Job receiver = factory.createReceiver(receiverCfg, engine); try { @@ -257,7 +255,7 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { } // warm writer up - try (TableWriter w = pool.get("tab")) { + try (TableWriter w = engine.getWriter("tab")) { w.warmUp(); } @@ -266,14 +264,14 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { Worker worker = new Worker(jobs, workerHaltLatch); worker.start(); - try (LineProtoSender sender = new LineProtoSender(NetworkFacadeImpl.INSTANCE, 0, Net.parseIPv4(receiverCfg.getBindIPv4Address()), receiverCfg.getPort(), 1400)) { + try (LineProtoSender sender = new LineProtoSender(NetworkFacadeImpl.INSTANCE, 0, receiverCfg.getBindIPv4Address(), receiverCfg.getPort(), 1400)) { for (int i = 0; i < 10; i++) { sender.metric("tab").tag("colour", "blue").tag("shape", "square").field("size", 3.4, 4).$(100000000); } sender.flush(); } - try (TableReader reader = new TableReader(cairoCfg, "tab")) { + try (TableReader reader = new TableReader(new DefaultCairoConfiguration(root), "tab")) { int count = 1000000; while (true) { if (count-- > 0 && reader.size() < 10) { @@ -300,11 +298,11 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { }); } - private static class TestReceiverConfiguration implements ReceiverConfiguration { + private static class TestLineUdpReceiverConfiguration implements LineUdpReceiverConfiguration { @Override - public CharSequence getBindIPv4Address() { - return "127.0.0.1"; + public int getBindIPv4Address() { + return Net.parseIPv4("127.0.0.1"); } @Override @@ -313,8 +311,8 @@ public class LinuxLineProtoReceiverTest extends AbstractCairoTest { } @Override - public CharSequence getGroupIPv4Address() { - return "224.1.1.1"; + public int getGroupIPv4Address() { + return Net.parseIPv4("224.1.1.1"); } @Override diff --git a/core/src/test/java/com/questdb/cutlass/text/TextLoaderTest.java b/core/src/test/java/com/questdb/cutlass/text/TextLoaderTest.java index dab7cfc86f4e09e41dbe4b95ef1dcd1f3cfe83dd..012df06fa913879a8bcad5f4fc6e9ea1ebb73811 100644 --- a/core/src/test/java/com/questdb/cutlass/text/TextLoaderTest.java +++ b/core/src/test/java/com/questdb/cutlass/text/TextLoaderTest.java @@ -619,12 +619,12 @@ public class TextLoaderTest extends AbstractCairoTest { try (TextLoader loader = new TextLoader( new DefaultTextConfiguration() { @Override - public long getRollBufferLimit() { + public int getRollBufferLimit() { return 128; } @Override - public long getRollBufferSize() { + public int getRollBufferSize() { return 32; } }, @@ -883,12 +883,12 @@ public class TextLoaderTest extends AbstractCairoTest { try (TextLoader loader = new TextLoader( new DefaultTextConfiguration() { @Override - public long getRollBufferLimit() { + public int getRollBufferLimit() { return 128; } @Override - public long getRollBufferSize() { + public int getRollBufferSize() { return 32; } }, diff --git a/core/src/test/resources/server.conf b/core/src/test/resources/server.conf new file mode 100644 index 0000000000000000000000000000000000000000..49954f51869b277826924a54210ebf6f584063ce --- /dev/null +++ b/core/src/test/resources/server.conf @@ -0,0 +1,78 @@ +http.connection.pool.initial.capacity=64 +http.connection.string.pool.capacity=512 +http.multipart.header.buffer.size=256 +http.multipart.idle.spin.count=100000 +http.receive.buffer.size=4k +http.request.header.buffer.size=2k +http.response.header.buffer.size=9012 +http.worker.count=6 +http.send.buffer.size=128 +http.static.index.file.name=index2.html +http.static.pubic.directory=public_ok +http.net.active.connection.limit=64 +http.net.event.capacity=2048 +http.net.io.queue.capacity=64 +http.net.idle.connection.timeout=7000000 +http.net.interest.queue.capacity=512 +http.net.listen.backlog=64 +http.net.snd.buf.size=4m +http.net.rcv.buf.size=8m + +http.text.abort.broken.uploads=false +http.text.adapter.set.config=/loader.json +http.text.date.adapter.pool.capacity=32 +http.text.json.cache.limit=64k +http.text.json.cache.size=8m +http.text.max.required.delimiter.stddev=0.3d +http.text.metadata.string.pool.capacity=512 +http.text.roll.buffer.limit=6k +http.text.roll.buffer.size=3k +http.text.analysis.max.lines=400 +http.text.lexer.string.pool.capacity=128 +http.text.timestamp.adapter.pool.capacity=512 +http.text.utf8.sink.size=8k +http.bind.to=10.5.8.30:9900 + +cairo.create.as.select.retry.count=12 +cairo.default.map.type=compact +cairo.default.symbol.cache.flag=true +cairo.default.symbol.capacity=512 +cairo.file.operation.retry.count=10 +cairo.idle.check.interval=20000 +cairo.inactive.reader.ttl=600000 +cairo.inactive.writer.ttl=400000 +cairo.index.value.block.size=1024 +cairo.max.swap.file.count=23 +cairo.mkdir.mode=580 +cairo.parallel.index.threshold=1000000 +cairo.reader.pool.max.segments=10 +cairo.spin.lock.timeout=5000000 +cairo.cache.rows=32 +cairo.cache.blocks=16 +cairo.character.store.capacity=2048 +cairo.character.store.sequence.pool.capacity=128 +cairo.column.pool.capacity=2048 +cairo.compact.map.load.factor=0.8 +cairo.expression.pool.capacity=1024 +cairo.fast.map.load.factor=0.3 +cairo.sql.join.context.pool.capacity=32 +cairo.lexer.pool.capacity=1024 +cairo.sql.map.key.capacity=1024 +cairo.sql.map.page.size=6m +cairo.model.pool.capacity=256 +cairo.sql.sort.key.page.size=10m +cairo.sql.sort.light.value.page.size=3m +cairo.sql.hash.join.value.page.size=8m +cairo.sql.tree.page.size=16m +cairo.sql.hash.join.light.value.page.size=2m +cairo.sql.sort.value.page.size=4m +cairo.work.steal.timeout.nanos=1000000 +cairo.parallel.indexing.enabled=false +cairo.sql.join.metadata.page.size=8k + +line.udp.bind.to=10.2.1.33:9915 +line.udp.commit.rate=100000 +line.udp.join=224.1.1.1 +line.udp.msg.buffer.size=4m +line.udp.msg.count=4000 +line.udp.receive.buffer.size=512