Merge pull request #2164 from topvisor/add_table_function_file

Add table function file

dbms/src/Storages/StorageFile.cpp

@@ -20,6 +20,8 @@
 
 #include <fcntl.h>
 
+#include <Poco/Path.h>
+#include <Poco/File.h>
 
 namespace DB
 {
@@ -32,6 +34,7 @@ namespace ErrorCodes
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int UNKNOWN_IDENTIFIER;
     extern const int INCORRECT_FILE_NAME;
+    extern const int FILE_DOESNT_EXIST;
     extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
 };
 
@@ -41,10 +44,22 @@ static std::string getTablePath(const std::string & db_dir_path, const std::stri
     return db_dir_path + escapeForFileName(table_name) + "/data." + escapeForFileName(format_name);
 }
 
-static void checkCreationIsAllowed(Context & context_global)
+/// Both db_dir_path and table_path must be converted to absolute paths (in particular, path cannot contain '..').
+static void checkCreationIsAllowed(Context & context_global, const std::string & db_dir_path, const std::string & table_path, int table_fd)
 {
-    if (context_global.getApplicationType() == Context::ApplicationType::SERVER)
-        throw Exception("Using file descriptor or user specified path as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED);
+    if (context_global.getApplicationType() != Context::ApplicationType::SERVER)
+        return;
+
+    if (table_fd >= 0)
+        throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED);
+    else if (!startsWith(table_path, db_dir_path))
+        throw Exception("Part path " + table_path + " is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED);
+
+    Poco::File table_path_poco_file = Poco::File(table_path);
+    if (!table_path_poco_file.exists())
+        throw Exception("File " + table_path + " is not exist", ErrorCodes::FILE_DOESNT_EXIST);
+    else if (table_path_poco_file.isDirectory())
+        throw Exception("File " + table_path + " must not be a directory", ErrorCodes::INCORRECT_FILE_NAME);
 }
 
 
@@ -65,8 +80,12 @@ StorageFile::StorageFile(
 
         if (!table_path_.empty()) /// Is user's file
         {
-            checkCreationIsAllowed(context_global);
-            path = Poco::Path(table_path_).absolute().toString();
+            Poco::Path poco_path = Poco::Path(table_path_);
+            if (poco_path.isRelative())
+                poco_path = Poco::Path(db_dir_path, poco_path);
+
+            path = poco_path.absolute().toString();
+            checkCreationIsAllowed(context_global, db_dir_path, path, table_fd);
             is_db_table = false;
         }
         else /// Is DB's file
@@ -81,7 +100,8 @@ StorageFile::StorageFile(
     }
     else /// Will use FD
     {
-        checkCreationIsAllowed(context_global);
+        checkCreationIsAllowed(context_global, db_dir_path, path, table_fd);
+
         is_db_table = false;
         use_table_fd = true;
 

dbms/src/TableFunctions/TableFunctionFile.cpp

+#include <TableFunctions/ITableFunction.h>
+#include <TableFunctions/TableFunctionFile.h>
+#include <TableFunctions/TableFunctionFactory.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTLiteral.h>
+#include <Common/Exception.h>
+#include <Common/typeid_cast.h>
+#include <Storages/StorageFile.h>
+#include <DataTypes/DataTypeFactory.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/evaluateConstantExpression.h>
+#include <boost/algorithm/string.hpp>
+
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+        extern const int DATABASE_ACCESS_DENIED;
+    }
+
+    StoragePtr TableFunctionFile::executeImpl(const ASTPtr & ast_function, const Context & context) const
+    {
+        // Parse args
+        ASTs & args_func = typeid_cast<ASTFunction &>(*ast_function).children;
+
+        if (args_func.size() != 1)
+            throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::LOGICAL_ERROR);
+
+        ASTs & args = typeid_cast<ASTExpressionList &>(*args_func.at(0)).children;
+
+        if (args.size() != 3)
+            throw Exception("Table function '" + getName() + "' requires exactly 3 arguments: path, format and structure.",
+                            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+        for (size_t i = 0; i < 3; ++i)
+            args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context);
+
+        std::string path = static_cast<const ASTLiteral &>(*args[0]).value.safeGet<String>();
+        std::string format = static_cast<const ASTLiteral &>(*args[1]).value.safeGet<String>();
+        std::string structure = static_cast<const ASTLiteral &>(*args[2]).value.safeGet<String>();
+
+        // Create sample block
+        std::vector<std::string> structure_vals;
+        boost::split(structure_vals, structure, boost::algorithm::is_any_of(" ,"), boost::algorithm::token_compress_on);
+
+        if (structure_vals.size() % 2 != 0)
+            throw Exception("Odd number of elements in section structure: must be a list of name type pairs", ErrorCodes::LOGICAL_ERROR);
+
+        Block sample_block;
+        const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
+
+        for (size_t i = 0, size = structure_vals.size(); i < size; i += 2)
+        {
+            ColumnWithTypeAndName column;
+            column.name = structure_vals[i];
+            column.type = data_type_factory.get(structure_vals[i + 1]);
+            column.column = column.type->createColumn();
+            sample_block.insert(std::move(column));
+        }
+
+        // Create table
+        StoragePtr storage = StorageFile::create(
+                path, -1, context.getPath() + "data/", getName(), format,
+                ColumnsDescription{sample_block.getNamesAndTypesList()}, const_cast<Context &>(context));
+
+        storage->startup();
+
+        return storage;
+    }
+
+
+    void registerTableFunctionFile(TableFunctionFactory & factory)
+    {
+        factory.registerFunction<TableFunctionFile>();
+    }
+
+}

dbms/src/TableFunctions/TableFunctionFile.h

+#pragma once
+
+#include <TableFunctions/ITableFunction.h>
+
+
+namespace DB
+{
+
+/* file(path, format, structure) - creates a temporary storage from file
+ *
+ *
+ * The file must be in the clickhouse data directory.
+ * The relative path begins with the clickhouse data directory.
+ */
+    class TableFunctionFile : public ITableFunction
+    {
+    public:
+        static constexpr auto name = "file";
+        std::string getName() const override { return name; }
+    private:
+        StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context) const override;
+    };
+
+
+}

dbms/src/TableFunctions/registerTableFunctions.cpp

@@ -11,6 +11,8 @@ void registerTableFunctionRemote(TableFunctionFactory & factory);
 void registerTableFunctionShardByHash(TableFunctionFactory & factory);
 void registerTableFunctionNumbers(TableFunctionFactory & factory);
 void registerTableFunctionCatBoostPool(TableFunctionFactory & factory);
+void registerTableFunctionFile(TableFunctionFactory & factory);
+
 #if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
 void registerTableFunctionODBC(TableFunctionFactory & factory);
 #endif
@@ -29,6 +31,7 @@ void registerTableFunctions()
     registerTableFunctionShardByHash(factory);
     registerTableFunctionNumbers(factory);
     registerTableFunctionCatBoostPool(factory);
+    registerTableFunctionFile(factory);
 
 #if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
     registerTableFunctionODBC(factory);