Added --stacktrace option to clickhouse-local. [#METR-23765]

Added support of path config variable and ability to load clickhouse-server data. [#METR-20000]
Default database of clickhouse-local is changed to "_local" to avoid collisions with clickhouse-server.

Small fixes and enhancements of command line options (setting and limits). [#METR-23101]

dbms/src/Client/Client.cpp

@@ -252,7 +252,7 @@ private:
 					<< e.getStackTrace().toString();
 			}
 
-			/// В случае нулевого кода исключения, надо всё-равно вернуть ненулевой код возврата.
+			/// If exception code isn't zero, we should return non-zero return code anyway.
 			return e.code() ? e.code() : -1;
 		}
 		catch (const Poco::Exception & e)

dbms/src/Server/LocalServer.cpp

@@ -2,6 +2,7 @@
 #include <Poco/Util/XMLConfiguration.h>
 #include <Poco/Util/HelpFormatter.h>
 #include <Poco/Util/OptionCallback.h>
+#include <Poco/String.h>
 
 #include <DB/Databases/DatabaseOrdinary.h>
 
@@ -19,7 +20,9 @@
 #include <DB/Interpreters/Context.h>
 #include <DB/Interpreters/ProcessList.h>
 #include <DB/Interpreters/executeQuery.h>
+#include <DB/Interpreters/loadMetadata.h>
 
+#include <DB/Common/Exception.h>
 #include <DB/Common/Macros.h>
 #include <DB/Common/ConfigProcessor.h>
 #include <DB/Common/escapeForFileName.h>
@@ -33,6 +36,8 @@
 #include <common/ErrorHandlers.h>
 #include <common/ApplicationServerExt.h>
 
+#include "StatusFile.h"
+
 
 namespace DB
 {
@@ -46,30 +51,28 @@ namespace ErrorCodes
 
 LocalServer::LocalServer() = default;
 
-LocalServer::~LocalServer() = default;
+LocalServer::~LocalServer()
+{
+	if (context)
+		context->shutdown(); /// required for properly exception handling
+}
 
 
 void LocalServer::initialize(Poco::Util::Application & self)
 {
 	Poco::Util::Application::initialize(self);
-
-	/// Load config files if exists
-	if (config().has("config-file") || Poco::File("config.xml").exists())
-	{
-		ConfigurationPtr processed_config = ConfigProcessor(false, true).loadConfig(config().getString("config-file", "config.xml"));
-		config().add(processed_config.duplicate(), PRIO_DEFAULT, false);
-	}
 }
 
+
 void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
 {
 	Poco::Util::Application::defineOptions (_options);
 
 	_options.addOption(
-		Poco::Util::Option ("config-file", "C", "Load configuration from a given file")
+		Poco::Util::Option ("config-file", "", "Load configuration from a given file")
 			.required(false)
 			.repeatable(false)
-			.argument(" config.xml")
+			.argument("[config.xml]")
 			.binding("config-file")
 			);
 
@@ -79,7 +82,7 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
 		Poco::Util::Option ("structure", "S", "Structe of initial table(list columns names with their types)")
 			.required(false)
 			.repeatable(false)
-			.argument(" <struct>")
+			.argument("[name Type]")
 			.binding("table-structure")
 			);
 
@@ -87,12 +90,12 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
 		Poco::Util::Option ("table", "N", "Name of intial table")
 			.required(false)
 			.repeatable(false)
-			.argument(" table")
+			.argument("[table]")
 			.binding("table-name")
 			);
 
 	_options.addOption(
-		Poco::Util::Option ("file", "F", "Path to file with data of initial table (stdin if not specified)")
+		Poco::Util::Option ("file", "f", "Path to file with data of initial table (stdin if not specified)")
 			.required(false)
 			.repeatable(false)
 			.argument(" stdin")
@@ -103,16 +106,16 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
 		Poco::Util::Option ("input-format", "if", "Input format of intial table data")
 			.required(false)
 			.repeatable(false)
-			.argument(" TabSeparated")
+			.argument("[TabSeparated]")
 			.binding("table-data-format")
 			);
 
 	/// List of queries to execute
 	_options.addOption(
-		Poco::Util::Option ("query", "Q", "Queries to execute")
+		Poco::Util::Option ("query", "q", "Queries to execute")
 			.required(false)
 			.repeatable(false)
-			.argument(" <query>", true)
+			.argument("<query>", true)
 			.binding("query")
 			);
 
@@ -121,19 +124,26 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
 		Poco::Util::Option ("output-format", "of", "Default output format")
 			.required(false)
 			.repeatable(false)
-			.argument(" TabSeparated", true)
+			.argument("[TabSeparated]", true)
 			.binding("output-format")
 			);
 
 	/// Alias for previous one, required for clickhouse-client compability
 	_options.addOption(
-		Poco::Util::Option ("format", "f", "Default ouput format")
+		Poco::Util::Option ("format", "", "Default ouput format")
 			.required(false)
 			.repeatable(false)
-			.argument(" TabSeparated", true)
+			.argument("[TabSeparated]", true)
 			.binding("format")
 			);
 
+	_options.addOption(
+		Poco::Util::Option ("stacktrace", "", "Print stack traces of exceptions")
+			.required(false)
+			.repeatable(false)
+			.binding("stacktrace")
+			);
+
 	_options.addOption(
 		Poco::Util::Option ("verbose", "", "Print info about execution of queries")
 			.required(false)
@@ -151,17 +161,13 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
 		.callback(Poco::Util::OptionCallback<LocalServer>(this, &LocalServer::handleHelp)));
 
 #define DECLARE_SETTING(TYPE, NAME, DEFAULT) \
-	{ \
-		_options.addOption(Poco::Util::Option(#NAME, "", "Settings.h").group("settings").required(false).repeatable(false).binding(#NAME)); \
-		APPLY_FOR_SETTINGS(DECLARE_SETTING) \
-	}
+		_options.addOption(Poco::Util::Option(#NAME, "", "Settings.h").group("Settings").required(false).repeatable(false).binding(#NAME));
+	APPLY_FOR_SETTINGS(DECLARE_SETTING)
 #undef DECLARE_SETTING
 
 #define DECLARE_SETTING(TYPE, NAME, DEFAULT) \
-	{ \
-		_options.addOption(Poco::Util::Option(#NAME, "", "Limits.h").group("limits").required(false).repeatable(false).binding(#NAME)); \
-		APPLY_FOR_LIMITS(DECLARE_SETTING) \
-	}
+		_options.addOption(Poco::Util::Option(#NAME, "", "Limits.h").group("Limits").required(false).repeatable(false).binding(#NAME));
+	APPLY_FOR_LIMITS(DECLARE_SETTING)
 #undef DECLARE_SETTING
 }
 
@@ -204,7 +210,7 @@ void LocalServer::displayHelp()
 	helpFormatter.format(std::cerr);
 	std::cerr << "Example printing memory used by each Unix user:\n"
 	"ps aux | tail -n +2 | awk '{ printf(\"%s\\t%s\\n\", $1, $4) }' | "
-	"clickhouse-local -S \"user String, mem Float64\" -Q \"SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty;\"\n";
+	"clickhouse-local -S \"user String, mem Float64\" -q \"SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty\"\n";
 }
 
 
@@ -215,19 +221,41 @@ void LocalServer::handleHelp(const std::string & name, const std::string & value
 }
 
 
+/// If path is specified and not empty, will try to setup server environment and load existing metadata
+void LocalServer::tryInitPath()
+{
+	if (!config().has("path") || (path = config().getString("path")).empty())
+		return;
+
+	Poco::trimInPlace(path);
+	if (path.empty())
+		return;
+	if (path.back() != '/')
+		path += '/';
+
+	context->setPath(path);
+
+	StatusFile status{path + "status"};
+}
+
+
 int LocalServer::main(const std::vector<std::string> & args)
+try
 {
+	Logger * log = &logger();
+
 	if (!config().has("query") && !config().has("table-structure")) /// Nothing to process
 	{
-		if (!config().has("help"))
+		if (!config().hasOption("help"))
 		{
-			std::cerr << "There are no queries to process.\n";
+			std::cerr << "There are no queries to process." << std::endl;
 			displayHelp();
 		}
 
 		return Application::EXIT_OK;
 	}
 
+	/// Load config files if exists
 	if (config().has("config-file") || Poco::File("config.xml").exists())
 	{
 		ConfigurationPtr processed_config = ConfigProcessor(false, true).loadConfig(config().getString("config-file", "config.xml"));
@@ -237,10 +265,11 @@ int LocalServer::main(const std::vector<std::string> & args)
 	context = std::make_unique<Context>();
 	context->setGlobalContext(*context);
 	context->setApplicationType(Context::ApplicationType::LOCAL_SERVER);
+	tryInitPath();
 
 	applyOptions();
 
-	/// Skip path, temp path, flag's path installation
+	/// Skip temp path installation
 
 	/// We will terminate process on error
 	static KillingErrorHandler error_handler;
@@ -274,10 +303,22 @@ int LocalServer::main(const std::vector<std::string> & args)
 	/// Load global settings from default profile.
 	context->setSetting("profile", config().getString("default_profile", "default"));
 
-	/// Init dummy default DB
-	const std::string default_database = "default";
+	/** Init dummy default DB
+	  * NOTE: We force using isolated default database to avoid conflicts with default database from server enviroment
+	  * Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory;
+	  *  if such tables will not be dropped, clickhouse-server can not load them due to security reasons.
+	  */
+	const std::string default_database = "_local";
 	context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
 	context->setCurrentDatabase(default_database);
+
+	if (!path.empty())
+	{
+		LOG_DEBUG(log, "Loading metadata from " << path);
+		loadMetadata(*context);
+		LOG_DEBUG(log, "Loaded metadata.");
+	}
+
 	attachSystemTables();
 
 	processQueries();
@@ -287,6 +328,26 @@ int LocalServer::main(const std::vector<std::string> & args)
 
 	return Application::EXIT_OK;
 }
+catch (const Exception & e)
+{
+	bool print_stack_trace = config().has("stacktrace");
+
+	std::string text = e.displayText();
+
+	auto embedded_stack_trace_pos = text.find("Stack trace");
+	if (std::string::npos != embedded_stack_trace_pos && !print_stack_trace)
+		text.resize(embedded_stack_trace_pos);
+
+	std::cerr << "Code: " << e.code() << ". " << text << std::endl << std::endl;
+
+	if (print_stack_trace && std::string::npos == embedded_stack_trace_pos)
+	{
+		std::cerr << "Stack trace:" << std::endl << e.getStackTrace().toString();
+	}
+
+	/// If exception code isn't zero, we should return non-zero return code anyway.
+	return e.code() ? e.code() : -1;
+}
 
 
 inline String getQuotedString(const String & s)
@@ -323,10 +384,14 @@ std::string LocalServer::getInitialCreateTableQuery()
 
 void LocalServer::attachSystemTables()
 {
-	/// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
+	DatabasePtr system_database = context->tryGetDatabase("system");
+	if (!system_database)
+	{
+		/// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
+		system_database = std::make_shared<DatabaseMemory>("system");
+		context->addDatabase("system", system_database);
+	}
 
-	DatabasePtr system_database = std::make_shared<DatabaseMemory>("system");
-	context->addDatabase("system", system_database);
 	system_database->attachTable("one", 	StorageSystemOne::create("one"));
 	system_database->attachTable("numbers", StorageSystemNumbers::create("numbers"));
 	system_database->attachTable("numbers_mt", StorageSystemNumbers::create("numbers_mt", true));
@@ -346,7 +411,7 @@ void LocalServer::processQueries()
 	String initial_create_query = getInitialCreateTableQuery();
 	String queries_str = initial_create_query + config().getString("query");
 
-	bool verbose = config().getBool("verbose", false);
+	bool verbose = config().hasOption("verbose");
 
 	std::vector<String> queries;
 	auto parse_res = splitMultipartQuery(queries_str, queries);
@@ -358,22 +423,14 @@ void LocalServer::processQueries()
 
 	for (const auto & query : queries)
 	{
-		try
-		{
-			ReadBufferFromString read_buf(query);
-			WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO);
-			BlockInputStreamPtr plan;
+		ReadBufferFromString read_buf(query);
+		WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO);
+		BlockInputStreamPtr plan;
 
-			if (verbose)
-				LOG_INFO(log, "Executing query: " << query);
+		if (verbose)
+			LOG_INFO(log, "Executing query: " << query);
 
-			executeQuery(read_buf, write_buf, *context, plan, nullptr);
-		}
-		catch (...)
-		{
-			tryLogCurrentException(log, "An error ocurred while executing query");
-			throw;
-		}
+		executeQuery(read_buf, write_buf, *context, plan, nullptr);
 	}
 }
 

dbms/src/Server/LocalServer.h

@@ -33,6 +33,8 @@ private:
 	  */
 	std::string getInitialCreateTableQuery();
 
+	void tryInitPath();
+
 	void applyOptions();
 
 	void attachSystemTables();
@@ -48,6 +50,7 @@ private:
 protected:
 
 	std::unique_ptr<Context> context;
+	std::string path;
 };
 
 }

dbms/src/Server/Server.cpp

@@ -178,16 +178,20 @@ public:
 };
 
 
-int Server::main(const std::vector<std::string> & args)
+static std::string getCanonicalPath(std::string && path)
 {
-	Logger * log = &logger();
-
-	std::string path = config().getString("path");
 	Poco::trimInPlace(path);
 	if (path.empty())
 		throw Exception("path configuration parameter is empty");
 	if (path.back() != '/')
 		path += '/';
+	return path;
+}
+
+
+int Server::main(const std::vector<std::string> & args)
+{
+	Logger * log = &logger();
 
 	/** Context contains all that query execution is dependent:
 	  *  settings, available functions, data types, aggregate functions, databases...
@@ -195,10 +199,12 @@ int Server::main(const std::vector<std::string> & args)
 	global_context = std::make_unique<Context>();
 	global_context->setGlobalContext(*global_context);
 	global_context->setApplicationType(Context::ApplicationType::SERVER);
-	global_context->setPath(path);
 
+	std::string path = getCanonicalPath(config().getString("path"));
 	std::string default_database = config().getString("default_database", "default");
 
+	global_context->setPath(path);
+
 	/// Create directories for 'path' and for default database, if not exist.
 	Poco::File(path + "data/" + default_database).createDirectories();
 	Poco::File(path + "metadata/" + default_database).createDirectories();