diff --git a/core/src/main/java/io/questdb/DefaultTelemetryConfiguration.java b/core/src/main/java/io/questdb/DefaultTelemetryConfiguration.java index 4065cd58da991b3c1add7865ede10f32193fcfe2..8be993b06ad27b0ddc8c657507a0621ffcf70c92 100644 --- a/core/src/main/java/io/questdb/DefaultTelemetryConfiguration.java +++ b/core/src/main/java/io/questdb/DefaultTelemetryConfiguration.java @@ -39,4 +39,9 @@ public class DefaultTelemetryConfiguration implements TelemetryConfiguration { public int getQueueCapacity() { return 16; } + + @Override + public boolean hideTables() { + return false; + } } diff --git a/core/src/main/java/io/questdb/PropServerConfiguration.java b/core/src/main/java/io/questdb/PropServerConfiguration.java index 9b9b70a1ab63c125dbe2b420bb6f08d193166f20..2b722b0b8f6bd042745644df5d0e2c0d8610d0c4 100644 --- a/core/src/main/java/io/questdb/PropServerConfiguration.java +++ b/core/src/main/java/io/questdb/PropServerConfiguration.java @@ -179,6 +179,7 @@ public class PropServerConfiguration implements ServerConfiguration { private final boolean telemetryEnabled; private final boolean telemetryDisableCompletely; private final int telemetryQueueCapacity; + private final boolean telemetryHideTables; private final LineTcpReceiverConfiguration lineTcpReceiverConfiguration = new PropLineTcpReceiverConfiguration(); private final IODispatcherConfiguration lineTcpReceiverDispatcherConfiguration = new PropLineTcpReceiverIODispatcherConfiguration(); private final boolean lineTcpEnabled; @@ -757,6 +758,7 @@ public class PropServerConfiguration implements ServerConfiguration { this.telemetryEnabled = getBoolean(properties, env, "telemetry.enabled", true); this.telemetryDisableCompletely = getBoolean(properties, env, "telemetry.disable.completely", false); this.telemetryQueueCapacity = Numbers.ceilPow2(getInt(properties, env, "telemetry.queue.capacity", 512)); + this.telemetryHideTables = getBoolean(properties, env, "telemetry.hide.tables", true); this.o3PartitionPurgeListCapacity = getInt(properties, env, "cairo.o3.partition.purge.list.initial.capacity", 1); parseBindTo(properties, env, "line.udp.bind.to", "0.0.0.0:9009", (a, p) -> { @@ -2914,6 +2916,11 @@ public class PropServerConfiguration implements ServerConfiguration { public int getQueueCapacity() { return telemetryQueueCapacity; } + + @Override + public boolean hideTables() { + return telemetryHideTables; + } } private class PropHttpMinServerConfiguration implements HttpMinServerConfiguration { diff --git a/core/src/main/java/io/questdb/TelemetryConfiguration.java b/core/src/main/java/io/questdb/TelemetryConfiguration.java index f4d5fe0059683269b4a71ab30f95e1b96f64826a..d45aed0eccd132e108dd7ae347b882f343739e59 100644 --- a/core/src/main/java/io/questdb/TelemetryConfiguration.java +++ b/core/src/main/java/io/questdb/TelemetryConfiguration.java @@ -30,4 +30,6 @@ public interface TelemetryConfiguration { boolean getEnabled(); int getQueueCapacity(); + + boolean hideTables(); } diff --git a/core/src/main/java/io/questdb/TelemetryJob.java b/core/src/main/java/io/questdb/TelemetryJob.java index d007b8f7e3ce5f697c152e7cb4029c14cd79b526..57834fca4974ea6b0ed93921f37f0d456e73716d 100644 --- a/core/src/main/java/io/questdb/TelemetryJob.java +++ b/core/src/main/java/io/questdb/TelemetryJob.java @@ -48,22 +48,21 @@ import org.jetbrains.annotations.Nullable; import java.io.Closeable; public class TelemetryJob extends SynchronizedJob implements Closeable { - private static final Log LOG = LogFactory.getLog(TelemetryJob.class); - private static final String WRITER_LOCK_REASON = "telemetryJob"; - - private final static CharSequence tableName = "telemetry"; - final static CharSequence configTableName = "telemetry_config"; + public final static CharSequence tableName = "telemetry"; + public final static CharSequence configTableName = "telemetry_config"; static final String QDB_PACKAGE = "QDB_PACKAGE"; static final String OS_NAME = "os.name"; + private static final Log LOG = LogFactory.getLog(TelemetryJob.class); + private static final String WRITER_LOCK_REASON = "telemetryJob"; private final MicrosecondClock clock; private final CairoConfiguration configuration; private final RingQueue queue; private final SCSequence subSeq; + private final SCSequence tempSequence = new SCSequence(); private boolean enabled; private TableWriter writer; private final QueueConsumer myConsumer = this::newRowConsumer; private TableWriter configWriter; - private final SCSequence tempSequence = new SCSequence(); public TelemetryJob(CairoEngine engine) throws SqlException { this(engine, null); @@ -127,20 +126,6 @@ public class TelemetryJob extends SynchronizedJob implements Closeable { } } - private void tryAddColumn(SqlCompiler compiler, SqlExecutionContext executionContext, CharSequence columnDetails) { - try { - CompiledQuery cc = compiler.compile( - "ALTER TABLE " + configTableName + " ADD COLUMN " + columnDetails, - executionContext - ); - try (QueryFuture execution = cc.execute(tempSequence)) { - execution.await(); - } - } catch (SqlException ex) { - LOG.info().$("Failed to alter telemetry table [table=").$(configTableName).$(",error=").$(ex.getFlyweightMessage()).I$(); - } - } - @Override public void close() { if (enabled) { @@ -162,42 +147,6 @@ public class TelemetryJob extends SynchronizedJob implements Closeable { return false; } - private TableWriter updateTelemetryConfig( - SqlCompiler compiler, - SqlExecutionContextImpl sqlExecutionContext, - boolean enabled - ) throws SqlException { - final TableWriter configWriter = compiler.getEngine().getWriter(AllowAllCairoSecurityContext.INSTANCE, configTableName, WRITER_LOCK_REASON); - final CompiledQuery cc = compiler.compile(configTableName + " LIMIT -1", sqlExecutionContext); - try (final RecordCursor cursor = cc.getRecordCursorFactory().getCursor(sqlExecutionContext)) { - if (cursor.hasNext()) { - final Record record = cursor.getRecord(); - final boolean _enabled = record.getBool(1); - Long256 l256 = record.getLong256A(0); - final CharSequence lastVersion = record.getSym(2); - - // if the configuration changed to enable or disable telemetry - // we need to update the table to reflect that - if (enabled != _enabled || !configuration.getBuildInformation().getQuestDbVersion().equals(lastVersion)) { - appendConfigRow(compiler, configWriter, l256, enabled); - LOG.advisory() - .$("instance config changes [id=").$256(l256.getLong0(), l256.getLong1(), 0, 0) - .$(", enabled=").$(enabled) - .$(']').$(); - } else { - LOG.advisory() - .$("instance [id=").$256(l256.getLong0(), l256.getLong1(), 0, 0) - .$(", enabled=").$(enabled) - .$(']').$(); - } - } else { - // if there are no record for telemetry id we need to create one using clocks - appendConfigRow(compiler, configWriter, null, enabled); - } - } - return configWriter; - } - private void appendConfigRow(SqlCompiler compiler, TableWriter configWriter, Long256 id, boolean enabled) { TableWriter.Row row = configWriter.newRow(); if (null == id) { @@ -253,4 +202,54 @@ public class TelemetryJob extends SynchronizedJob implements Closeable { .$(']').$(); } } + + private void tryAddColumn(SqlCompiler compiler, SqlExecutionContext executionContext, CharSequence columnDetails) { + try { + CompiledQuery cc = compiler.compile( + "ALTER TABLE " + configTableName + " ADD COLUMN " + columnDetails, + executionContext + ); + try (QueryFuture execution = cc.execute(tempSequence)) { + execution.await(); + } + } catch (SqlException ex) { + LOG.info().$("Failed to alter telemetry table [table=").$(configTableName).$(",error=").$(ex.getFlyweightMessage()).I$(); + } + } + + private TableWriter updateTelemetryConfig( + SqlCompiler compiler, + SqlExecutionContextImpl sqlExecutionContext, + boolean enabled + ) throws SqlException { + final TableWriter configWriter = compiler.getEngine().getWriter(AllowAllCairoSecurityContext.INSTANCE, configTableName, WRITER_LOCK_REASON); + final CompiledQuery cc = compiler.compile(configTableName + " LIMIT -1", sqlExecutionContext); + try (final RecordCursor cursor = cc.getRecordCursorFactory().getCursor(sqlExecutionContext)) { + if (cursor.hasNext()) { + final Record record = cursor.getRecord(); + final boolean _enabled = record.getBool(1); + Long256 l256 = record.getLong256A(0); + final CharSequence lastVersion = record.getSym(2); + + // if the configuration changed to enable or disable telemetry + // we need to update the table to reflect that + if (enabled != _enabled || !configuration.getBuildInformation().getQuestDbVersion().equals(lastVersion)) { + appendConfigRow(compiler, configWriter, l256, enabled); + LOG.advisory() + .$("instance config changes [id=").$256(l256.getLong0(), l256.getLong1(), 0, 0) + .$(", enabled=").$(enabled) + .$(']').$(); + } else { + LOG.advisory() + .$("instance [id=").$256(l256.getLong0(), l256.getLong1(), 0, 0) + .$(", enabled=").$(enabled) + .$(']').$(); + } + } else { + // if there are no record for telemetry id we need to create one using clocks + appendConfigRow(compiler, configWriter, null, enabled); + } + } + return configWriter; + } } diff --git a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java index 151474c7bc50e37375e48f97b7b7fea1423c8fc9..8610c6cb91dcfa911408ec0d292df3715fdbc9ad 100644 --- a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java @@ -24,9 +24,10 @@ package io.questdb.griffin.engine.functions.catalogue; +import io.questdb.TelemetryJob; import io.questdb.cairo.*; -import io.questdb.cairo.sql.*; import io.questdb.cairo.sql.Record; +import io.questdb.cairo.sql.*; import io.questdb.griffin.FunctionFactory; import io.questdb.griffin.SqlExecutionContext; import io.questdb.griffin.engine.functions.CursorFunction; @@ -67,7 +68,10 @@ public class TableMetadataCursorFactory implements FunctionFactory { SqlExecutionContext sqlExecutionContext ) { return new CursorFunction( - new TableMetadataCursor(configuration.getFilesFacade(), configuration.getRoot()) + new TableMetadataCursor( + configuration.getFilesFacade(), + configuration.getRoot(), + configuration.getTelemetryConfiguration().hideTables()) ) { @Override public boolean isRuntimeConstant() { @@ -79,12 +83,14 @@ public class TableMetadataCursorFactory implements FunctionFactory { private static class TableMetadataCursor implements RecordCursorFactory { private final FilesFacade ff; private final TableListRecordCursor cursor; + private final boolean hideTelemetryTables; private Path path; - public TableMetadataCursor(FilesFacade ff, CharSequence dbRoot) { + public TableMetadataCursor(FilesFacade ff, CharSequence dbRoot, boolean hideTelemetryTables) { this.ff = ff; path = new Path().of(dbRoot).$(); cursor = new TableListRecordCursor(); + this.hideTelemetryTables = hideTelemetryTables; } @Override @@ -234,6 +240,11 @@ public class TableMetadataCursorFactory implements FunctionFactory { } public boolean open(CharSequence tableName) { + + if (hideTelemetryTables && (Chars.equals(tableName, TelemetryJob.tableName) || Chars.equals(tableName, TelemetryJob.configTableName))) { + return false; + } + int pathLen = path.length(); try { path.chop$().concat(tableName).concat(META_FILE_NAME).$(); diff --git a/core/src/main/resources/io/questdb/site/conf/server.conf b/core/src/main/resources/io/questdb/site/conf/server.conf index 8a97d0a2ae20ad0be3d569a8c15247ca7bf6756a..b5a5b354f9ed238e73caadbd478dfbec2885e316 100644 --- a/core/src/main/resources/io/questdb/site/conf/server.conf +++ b/core/src/main/resources/io/questdb/site/conf/server.conf @@ -509,9 +509,20 @@ query.timeout.sec=60 ################ Telemetry settings ################## +# Telemetry switch. Telemetry events are use to identify components of questdb that are being used. They never identify +# user not data stored in questdb. All events can be viewed via `select * from telemetry`. Switching telemetry off will +# stop all events from being collected and subsequent growth of `telemetry` table. After telemetry is switched off +# `telemetry` table cab be dropped. #telemetry.enabled=true + +# Capacity of the internal telemetry queue, which is the gateway of all telemetry events. +# This queue capacity does not require tweaking. #telemetry.queue.capacity=512 +# hides telemetry tables from `select * from tables()` output. As a result, telemetry tables +# will not be visible in the Web Console table view +#telemetry.hide.tables=true + ################ Metrics settings ################## #metrics.enabled=true diff --git a/core/src/test/java/io/questdb/cairo/AbstractCairoTest.java b/core/src/test/java/io/questdb/cairo/AbstractCairoTest.java index 01a326bb01ef0930aed636081990280f8877fb34..d7334c9225d174c36c4d7fbb734c3886d0ae7443 100644 --- a/core/src/test/java/io/questdb/cairo/AbstractCairoTest.java +++ b/core/src/test/java/io/questdb/cairo/AbstractCairoTest.java @@ -24,8 +24,10 @@ package io.questdb.cairo; +import io.questdb.DefaultTelemetryConfiguration; import io.questdb.MessageBus; import io.questdb.Metrics; +import io.questdb.TelemetryConfiguration; import io.questdb.cairo.sql.RecordCursor; import io.questdb.cairo.sql.RecordMetadata; import io.questdb.griffin.DatabaseSnapshotAgent; @@ -81,6 +83,8 @@ public class AbstractCairoTest { public static long writerAsyncCommandBusyWaitTimeout = -1; public static long writerAsyncCommandMaxTimeout = -1; public static long spinLockTimeoutUs = -1; + protected static boolean hideTelemetryTable = false; + private static TelemetryConfiguration telemetryConfiguration; @BeforeClass public static void setUpStatic() { @@ -94,6 +98,14 @@ public class AbstractCairoTest { } catch (IOException e) { throw new ExceptionInInitializerError(); } + + telemetryConfiguration = new DefaultTelemetryConfiguration() { + @Override + public boolean hideTables() { + return hideTelemetryTable; + } + }; + configuration = new DefaultCairoConfiguration(root) { @Override public FilesFacade getFilesFacade() { @@ -211,6 +223,11 @@ public class AbstractCairoTest { public boolean isSnapshotRecoveryEnabled() { return snapshotRecoveryEnabled == null ? super.isSnapshotRecoveryEnabled() : snapshotRecoveryEnabled; } + + @Override + public TelemetryConfiguration getTelemetryConfiguration() { + return telemetryConfiguration; + } }; engine = new CairoEngine(configuration, metrics); snapshotAgent = new DatabaseSnapshotAgent(engine); @@ -251,6 +268,7 @@ public class AbstractCairoTest { spinLockTimeoutUs = -1; snapshotInstanceId = null; snapshotRecoveryEnabled = null; + hideTelemetryTable = false; } protected static void assertMemoryLeak(TestUtils.LeakProneCode code) throws Exception { diff --git a/core/src/test/java/io/questdb/griffin/HideTelemetryTablesTest.java b/core/src/test/java/io/questdb/griffin/HideTelemetryTablesTest.java new file mode 100644 index 0000000000000000000000000000000000000000..9dc87aefae896e2b4191e986aa8e069ae167a455 --- /dev/null +++ b/core/src/test/java/io/questdb/griffin/HideTelemetryTablesTest.java @@ -0,0 +1,72 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2022 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.griffin; + +import io.questdb.TelemetryJob; +import io.questdb.test.tools.TestUtils; +import org.junit.Test; + +public class HideTelemetryTablesTest extends AbstractGriffinTest { + + @Test + public void testShow() throws Exception { + + assertMemoryLeak(() -> { + compiler.compile("create table test(a int)", sqlExecutionContext); + compiler.compile("create table " + TelemetryJob.tableName + "(a int)", sqlExecutionContext); + compiler.compile("create table " + TelemetryJob.configTableName + "(a int)", sqlExecutionContext); + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "tables order by 2", + sink, + "id\tname\tdesignatedTimestamp\tpartitionBy\tmaxUncommittedRows\tcommitLag\n" + + "2\ttelemetry\t\tNONE\t1000\t0\n" + + "3\ttelemetry_config\t\tNONE\t1000\t0\n" + + "1\ttest\t\tNONE\t1000\t0\n" + ); + }); + } + + @Test + public void testHide() throws Exception { + + hideTelemetryTable = true; + + assertMemoryLeak(() -> { + compiler.compile("create table test(a int)", sqlExecutionContext); + compiler.compile("create table " + TelemetryJob.tableName + "(a int)", sqlExecutionContext); + compiler.compile("create table " + TelemetryJob.configTableName + "(a int)", sqlExecutionContext); + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "tables", + sink, + "id\tname\tdesignatedTimestamp\tpartitionBy\tmaxUncommittedRows\tcommitLag\n" + + "1\ttest\t\tNONE\t1000\t0\n" + ); + }); + } +}