ISSUES-5436 support custom http [part 5]

dbms/src/Interpreters/CustomHTTP/CustomExecutor.cpp

 #include <Interpreters/CustomHTTP/CustomExecutor.h>
 #include <Interpreters/CustomHTTP/HTTPOutputStreams.h>
-#include <Interpreters/CustomHTTP/CustomExecutorDefault.h>
+#include "CustomExecutor.h"
 
 
 namespace DB
@@ -11,11 +11,11 @@ namespace ErrorCodes
     extern const int SYNTAX_ERROR;
 }
 
-bool CustomExecutor::match(HTTPRequest & request, HTMLForm & params) const
+bool CustomExecutor::match(Context & context, HTTPRequest & request, HTMLForm & params) const
 {
     for (const auto & matcher : matchers)
     {
-        if (!matcher->match(request, params))
+        if (!matcher->match(context, request, params))
             return false;
     }
 
@@ -56,14 +56,17 @@ void CustomExecutor::executeQuery(
 }
 
 CustomExecutor::CustomExecutor(
-    const std::vector<CustomMatcherPtr> & matchers_, const std::vector<CustomQueryExecutorPtr> & query_executors_)
+    const std::vector<CustomExecutorMatcherPtr> & matchers_, const std::vector<CustomQueryExecutorPtr> & query_executors_)
     : matchers(matchers_), query_executors(query_executors_)
 {
 }
 
-CustomExecutors::CustomExecutors(const Configuration & config, const Settings & settings, const String & config_prefix)
+static CustomExecutorPtr createDefaultCustomExecutor()
 {
-    updateCustomExecutors(config, settings, config_prefix);
+    std::vector<CustomExecutorMatcherPtr> custom_matchers{std::make_shared<AlwaysMatchedCustomExecutorMatcher>()};
+    std::vector<CustomQueryExecutorPtr> custom_query_executors{std::make_shared<ExtractQueryParamCustomQueryExecutor>()};
+
+    return std::make_shared<CustomExecutor>(custom_matchers, custom_query_executors);
 }
 
 void CustomExecutors::updateCustomExecutors(const Configuration & config, const Settings & settings, const String & config_prefix)
@@ -71,7 +74,7 @@ void CustomExecutors::updateCustomExecutors(const Configuration & config, const
     Configuration::Keys custom_executors_keys;
     config.keys(config_prefix, custom_executors_keys);
 
-    std::unordered_map<String, CustomExecutorPtr> new_custom_executors;
+    std::vector<std::pair<String, CustomExecutorPtr>> new_custom_executors;
 
     for (const auto & custom_executor_key : custom_executors_keys)
     {
@@ -80,20 +83,48 @@ void CustomExecutors::updateCustomExecutors(const Configuration & config, const
         else if (custom_executor_key.find('.') != String::npos)
             throw Exception("CustomExecutor names with dots are not supported: '" + custom_executor_key + "'", ErrorCodes::SYNTAX_ERROR);
 
-        new_custom_executors[custom_executor_key] = createCustomExecutor(config, settings, config_prefix + "." + custom_executor_key);
+        new_custom_executors.push_back(
+            std::make_pair(custom_executor_key, createCustomExecutor(config, config_prefix + "." + custom_executor_key)));
     }
 
-    new_custom_executors["Default"] = CustomExecutorDefault::createDefaultCustomExecutor();
+    new_custom_executors.push_back(std::make_pair("Default", createDefaultCustomExecutor()));
 
     std::unique_lock<std::shared_mutex> lock(rwlock);
     custom_executors = new_custom_executors;
 }
 
-CustomExecutorPtr CustomExecutors::createCustomExecutor(const CustomExecutors::Configuration & config, const Settings & /*settings*/, const String & config_prefix)
+void CustomExecutors::registerQueryExecutor(const String & query_executor_name, const CustomExecutors::QueryExecutorCreator & creator)
+{
+    const auto & matcher_creator_it = custom_matcher_creators.find(query_executor_name);
+    const auto & query_executor_creator_it = query_executor_creators.find(query_executor_name);
+
+    if (matcher_creator_it != custom_matcher_creators.end() && query_executor_creator_it != query_executor_creators.end())
+        throw Exception("LOGICAL_ERROR CustomQueryExecutor name must be unique between the CustomQueryExecutor and CustomExecutorMatcher.",
+                        ErrorCodes::LOGICAL_ERROR);
+
+    query_executor_creators[query_executor_name] = creator;
+}
+
+void CustomExecutors::registerCustomMatcher(const String & matcher_name, const CustomExecutors::CustomMatcherCreator & creator)
+{
+    const auto & matcher_creator_it = custom_matcher_creators.find(matcher_name);
+    const auto & query_executor_creator_it = query_executor_creators.find(matcher_name);
+
+    if (matcher_creator_it != custom_matcher_creators.end() && query_executor_creator_it != query_executor_creators.end())
+        throw Exception("LOGICAL_ERROR CustomExecutorMatcher name must be unique between the CustomQueryExecutor and CustomExecutorMatcher.",
+                        ErrorCodes::LOGICAL_ERROR);
+
+    custom_matcher_creators[matcher_name] = creator;
+}
+
+CustomExecutorPtr CustomExecutors::createCustomExecutor(const Configuration & config, const String & config_prefix)
 {
     Configuration::Keys matchers_or_query_executors_type;
     config.keys(config_prefix, matchers_or_query_executors_type);
 
+    std::vector<CustomQueryExecutorPtr> custom_query_executors;
+    std::vector<CustomExecutorMatcherPtr> custom_executor_matchers;
+
     for (const auto & matcher_or_query_executor_type : matchers_or_query_executors_type)
     {
         if (matcher_or_query_executor_type.find('.') != String::npos)
@@ -101,21 +132,43 @@ CustomExecutorPtr CustomExecutors::createCustomExecutor(const CustomExecutors::C
                 "CustomMatcher or CustomQueryExecutor names with dots are not supported: '" + matcher_or_query_executor_type + "'",
                 ErrorCodes::SYNTAX_ERROR);
 
-//        throw Exception("", ErrorCodes::NOT_IMPLEMENTED);
-//        new_custom_executors[matcher_or_query_executor_type] = createCustomExecutor(config, settings, config_prefix + "." + matcher_or_query_executor_type);
+        const auto & matcher_creator_it = custom_matcher_creators.find(matcher_or_query_executor_type);
+        const auto & query_executor_creator_it = query_executor_creators.find(matcher_or_query_executor_type);
+
+        if (matcher_creator_it == custom_matcher_creators.end() && query_executor_creator_it == query_executor_creators.end())
+            throw Exception("CustomMatcher or CustomQueryExecutor '" + matcher_or_query_executor_type + "' is not implemented.",
+                            ErrorCodes::NOT_IMPLEMENTED);
+
+        if (matcher_creator_it != custom_matcher_creators.end())
+            custom_executor_matchers.push_back(matcher_creator_it->second(config, config_prefix + "." + matcher_or_query_executor_type));
+
+        if (query_executor_creator_it != query_executor_creators.end())
+            custom_query_executors.push_back(query_executor_creator_it->second(config, config_prefix + "." + matcher_or_query_executor_type));
     }
-    return DB::CustomExecutorPtr();
+
+    for (const auto & custom_executor_matcher : custom_executor_matchers)
+        custom_executor_matcher->checkQueryExecutor(custom_query_executors);
+
+    return std::make_shared<CustomExecutor>(custom_executor_matchers, custom_query_executors);
 }
 
-std::pair<String, CustomExecutorPtr> CustomExecutors::getCustomExecutor(Poco::Net::HTTPServerRequest & request, HTMLForm & params) const
+std::pair<String, CustomExecutorPtr> CustomExecutors::getCustomExecutor(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params) const
 {
     std::shared_lock<std::shared_mutex> lock(rwlock);
 
     for (const auto & custom_executor : custom_executors)
-        if (custom_executor.second->match(request, params))
+        if (custom_executor.second->match(context, request, params))
             return custom_executor;
 
     throw Exception("LOGICAL_ERROR not found custom executor.", ErrorCodes::LOGICAL_ERROR);
 }
 
+CustomExecutors::CustomExecutors(const Configuration & config, const Settings & settings, const String & config_prefix)
+{
+    registerCustomMatcher("URL", [&](const auto & config, const auto & prefix)
+        { return std::make_shared<HTTPURLCustomExecutorMatcher>(config, prefix); });
+
+    updateCustomExecutors(config, settings, config_prefix);
+}
+
 }

dbms/src/Interpreters/CustomHTTP/CustomExecutor.h

@@ -10,14 +10,15 @@
 #include <Poco/Net/HTTPServerRequest.h>
 #include <Poco/Net/HTTPServerResponse.h>
 #include <Poco/Util/AbstractConfiguration.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/CustomHTTP/HTTPInputStreams.h>
+#include <Interpreters/CustomHTTP/HTTPOutputStreams.h>
+#include <Interpreters/CustomHTTP/CustomQueryExecutors.h>
+#include <Interpreters/CustomHTTP/CustomExecutorMatchers.h>
 
 namespace DB
 {
-
-class Context;
 class CustomExecutor;
-struct HTTPInputStreams;
-struct HTTPOutputStreams;
 
 using HTTPRequest = Poco::Net::HTTPServerRequest;
 using HTTPResponse = Poco::Net::HTTPServerResponse;
@@ -32,14 +33,22 @@ public:
     CustomExecutors(const CustomExecutors &) = delete;
     CustomExecutors & operator=(const CustomExecutors &) = delete;
 
+    using QueryExecutorCreator = std::function<CustomQueryExecutorPtr(const Configuration &, const String &)>;
+    void registerQueryExecutor(const String & query_executor_name, const QueryExecutorCreator & creator);
+
+    using CustomMatcherCreator = const std::function<CustomExecutorMatcherPtr(const Configuration &, const String &)>;
+    void registerCustomMatcher(const String & matcher_name, const CustomMatcherCreator & creator);
+
     void updateCustomExecutors(const Configuration & config, const Settings & settings, const String & config_prefix);
 
-    std::pair<String, CustomExecutorPtr> getCustomExecutor(Poco::Net::HTTPServerRequest & request, HTMLForm & params) const;
+    std::pair<String, CustomExecutorPtr> getCustomExecutor(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params) const;
 private:
     mutable std::shared_mutex rwlock;
-    std::unordered_map<String, CustomExecutorPtr> custom_executors;
+    std::vector<std::pair<String, CustomExecutorPtr>> custom_executors;
+    std::unordered_map<String, QueryExecutorCreator> query_executor_creators;
+    std::unordered_map<String, CustomMatcherCreator> custom_matcher_creators;
 
-    CustomExecutorPtr createCustomExecutor(const Configuration & config, const Settings & settings, const String & config_prefix);
+    CustomExecutorPtr createCustomExecutor(const Configuration & config, const String & config_prefix);
 };
 
 class CustomExecutor
@@ -47,45 +56,19 @@ class CustomExecutor
 public:
     bool isQueryParam(const String & param_name) const;
 
-    bool match(HTTPRequest & request, HTMLForm & params) const;
-
     bool canBeParseRequestBody(HTTPRequest & request, HTMLForm & params) const;
 
+    bool match(Context & context, HTTPRequest & request, HTMLForm & params) const;
+
     void executeQuery(
         Context & context, HTTPRequest & request, HTTPResponse & response,
         HTMLForm & params, const HTTPInputStreams & input_streams, const HTTPOutputStreams & output_streams
     );
 
-public:
-    class CustomMatcher
-    {
-    public:
-        virtual ~CustomMatcher() = default;
-
-        virtual bool match(HTTPRequest & request, HTMLForm & params) const = 0;
-    };
-
-    class CustomQueryExecutor
-    {
-    public:
-        virtual ~CustomQueryExecutor() = default;
-
-        virtual bool isQueryParam(const String &) const = 0;
-        virtual bool canBeParseRequestBody(HTTPRequest &, HTMLForm &) const = 0;
-
-        virtual void executeQueryImpl(
-            Context & context, HTTPRequest & request, HTTPResponse & response,
-            HTMLForm & params, const HTTPInputStreams & input_streams, const HTTPOutputStreams & output_streams) const = 0;
-    };
-
-public:
-    using CustomMatcherPtr = std::shared_ptr<CustomMatcher>;
-    using CustomQueryExecutorPtr = std::shared_ptr<CustomQueryExecutor>;
-
-    CustomExecutor(const std::vector<CustomMatcherPtr> & matchers_, const std::vector<CustomQueryExecutorPtr> & query_executors_);
+    CustomExecutor(const std::vector<CustomExecutorMatcherPtr> & matchers_, const std::vector<CustomQueryExecutorPtr> & query_executors_);
 
 private:
-    std::vector<CustomMatcherPtr> matchers;
+    std::vector<CustomExecutorMatcherPtr> matchers;
     std::vector<CustomQueryExecutorPtr> query_executors;
 };
 

dbms/src/Interpreters/CustomHTTP/CustomExecutorMatchers.h

+#pragma once
+
+#include <Core/Types.h>
+#include <Common/config.h>
+#include <Common/HTMLForm.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/CustomHTTP/CustomQueryExecutors.h>
+#include <Poco/Net/HTTPServerRequest.h>
+
+
+#if USE_RE2_ST
+#    include <re2_st/re2.h>
+#else
+#    define re2_st re2
+#endif
+
+namespace DB
+{
+
+class CustomExecutorMatcher
+{
+public:
+    virtual ~CustomExecutorMatcher() = default;
+
+    virtual bool checkQueryExecutor(const std::vector<CustomQueryExecutorPtr> & check_executors) const = 0;
+
+    virtual bool match(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params) const = 0;
+};
+
+using CustomExecutorMatcherPtr = std::shared_ptr<CustomExecutorMatcher>;
+
+
+class AlwaysMatchedCustomExecutorMatcher : public CustomExecutorMatcher
+{
+public:
+    bool checkQueryExecutor(const std::vector<CustomQueryExecutorPtr> & /*check_executors*/) const override { return true; }
+
+    bool match(Context & /*context*/, Poco::Net::HTTPServerRequest & /*request*/, HTMLForm & /*params*/) const override { return true; }
+};
+
+class HTTPURLCustomExecutorMatcher : public CustomExecutorMatcher
+{
+public:
+    HTTPURLCustomExecutorMatcher(const Poco::Util::AbstractConfiguration & configuration, const String & url_config_key)
+        : url_match_searcher(analyzeURLPatten(configuration.getString(url_config_key, ""), params_name_extract_from_url))
+    {
+    }
+
+    bool checkQueryExecutor(const std::vector<CustomQueryExecutorPtr> & custom_query_executors) const override
+    {
+        for (const auto & param_name_from_url : params_name_extract_from_url)
+        {
+            bool found_param_name = false;
+            for (const auto & custom_query_executor : custom_query_executors)
+            {
+                if (custom_query_executor->isQueryParam(param_name_from_url))
+                {
+                    found_param_name = true;
+                    break;
+                }
+            }
+
+            if (!found_param_name)
+                throw Exception("The param name '" + param_name_from_url + "' is uselessed.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        }
+
+        return true;
+    }
+
+    bool match(Context & /*context*/, Poco::Net::HTTPServerRequest & request, HTMLForm & /*params*/) const override
+    {
+        const String request_uri = request.getURI();
+        re2_st::StringPiece query_params_matches[params_name_extract_from_url.size()];
+
+//        re2_st::StringPiece input;
+//        if (url_match_searcher.Match(input, start_pos, input.length(), re2_st::RE2::Anchor::UNANCHORED, query_params_matches, num_captures))
+//        {
+//
+//        }
+        return false;
+    }
+
+private:
+    re2_st::RE2 url_match_searcher;
+    std::vector<String> params_name_extract_from_url;
+
+    String analyzeURLPatten(const String & /*url_patten*/, std::vector<String> & /*matches*/)
+    {
+        return ".+";
+        /// TODO: first we replace all capture group
+        /// TODO: second we replace all ${identifier}
+    }
+};
+
+
+}

dbms/src/Interpreters/CustomHTTP/CustomExecutorDefault.h → dbms/src/Interpreters/CustomHTTP/CustomQueryExecutors.h

 #pragma once
 
-#include <Interpreters/CustomHTTP/CustomExecutor.h>
+#include <Core/Types.h>
+#include <Common/HTMLForm.h>
 #include <IO/ConcatReadBuffer.h>
 #include <IO/ReadBufferFromString.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeQuery.h>
 #include <Interpreters/CustomHTTP/HTTPInputStreams.h>
 #include <Interpreters/CustomHTTP/HTTPOutputStreams.h>
-
+#include <Poco/Net/HTTPServerRequest.h>
+#include <Poco/Net/HTTPServerResponse.h>
 
 namespace DB
 {
 
-class CustomExecutorDefault : public CustomExecutor::CustomMatcher, public CustomExecutor::CustomQueryExecutor
+class CustomQueryExecutor
 {
 public:
-    bool match(HTTPServerRequest & /*request*/, HTMLForm & /*params*/) const override { return true; }
+    virtual ~CustomQueryExecutor() = default;
+
+    virtual bool isQueryParam(const String &) const = 0;
+    virtual bool canBeParseRequestBody(Poco::Net::HTTPServerRequest &, HTMLForm &) const = 0;
+
+    virtual void executeQueryImpl(
+        Context & context, Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response,
+        HTMLForm & params, const HTTPInputStreams & input_streams, const HTTPOutputStreams & output_streams) const = 0;
+};
 
-    bool canBeParseRequestBody(HTTPServerRequest & /*request*/, HTMLForm & /*params*/) const override { return false; }
+using CustomQueryExecutorPtr = std::shared_ptr<CustomQueryExecutor>;
 
+class ExtractQueryParamCustomQueryExecutor : public CustomQueryExecutor
+{
+public:
     bool isQueryParam(const String & param_name) const override { return param_name == "query" || startsWith(param_name, "param_"); }
 
+    bool canBeParseRequestBody(Poco::Net::HTTPServerRequest & /*request*/, HTMLForm & /*form*/) const override { return false; }
+
     void executeQueryImpl(
-        Context & context, HTTPRequest & request, HTTPResponse & response,
+        Context & context, Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response,
         HTMLForm & params, const HTTPInputStreams & input_streams, const HTTPOutputStreams & output_streams) const override
     {
         const auto & execute_query = prepareQuery(context, params);
@@ -42,16 +57,6 @@ public:
         );
     }
 
-    static CustomExecutorPtr createDefaultCustomExecutor()
-    {
-        const auto & default_custom_executor = std::make_shared<CustomExecutorDefault>();
-
-        std::vector<CustomExecutor::CustomMatcherPtr> custom_matchers{default_custom_executor};
-        std::vector<CustomExecutor::CustomQueryExecutorPtr> custom_query_executors{default_custom_executor};
-
-        return std::make_shared<CustomExecutor>(custom_matchers, custom_query_executors);
-    }
-
 private:
     String prepareQuery(Context & context, HTMLForm & params) const
     {

programs/server/HTTPHandler.cpp

@@ -244,7 +244,7 @@ void HTTPHandler::SessionContextHolder::authentication(HTTPServerRequest & reque
 void HTTPHandler::processQuery(Context & context, HTTPRequest & request, HTMLForm & params, HTTPResponse & response)
 {
     const auto & name_with_custom_executor = context.getCustomExecutor(request, params);
-    LOG_TRACE(log, "Using " << name_with_custom_executor.first << " to execute URI: " << request.getURI());
+    LOG_TRACE(log, "Using '" << name_with_custom_executor.first << "' CustomExecutor to execute URI: " << request.getURI());
 
     ExtractorClientInfo{context.getClientInfo()}.extract(request);
     ExtractorContextChange{context, name_with_custom_executor.second}.extract(request, params);

src/Interpreters/Context.cpp

@@ -2059,7 +2059,7 @@ void Context::setCustomExecutorConfig(const ConfigurationPtr & config, const Str
         shared->custom_executors->updateCustomExecutors(*shared->custom_executors_config, settings, config_prefix);
 }
 
-std::pair<String, CustomExecutorPtr> Context::getCustomExecutor(Poco::Net::HTTPServerRequest & request, HTMLForm & params) const
+std::pair<String, CustomExecutorPtr> Context::getCustomExecutor(Poco::Net::HTTPServerRequest & request, HTMLForm & params)
 {
     std::lock_guard lock(shared->custom_executors_mutex);
 
@@ -2069,7 +2069,7 @@ std::pair<String, CustomExecutorPtr> Context::getCustomExecutor(Poco::Net::HTTPS
         shared->custom_executors = std::make_unique<CustomExecutors>(config, settings);
     }
 
-    return shared->custom_executors->getCustomExecutor(request, params);
+    return shared->custom_executors->getCustomExecutor(*this, request, params);
 }
 
 

src/Interpreters/Context.h

@@ -42,7 +42,7 @@ namespace zkutil
     class ZooKeeper;
 }
 
-class HTMLForm;
+struct HTMLForm;
 
 namespace DB
 {
@@ -492,7 +492,7 @@ public:
     Compiler & getCompiler();
 
     void setCustomExecutorConfig(const ConfigurationPtr & config, const String & config_prefix = "CustomHTTP");
-    std::pair<String, CustomExecutorPtr> getCustomExecutor(Poco::Net::HTTPServerRequest &request, HTMLForm & params) const;
+    std::pair<String, CustomExecutorPtr> getCustomExecutor(Poco::Net::HTTPServerRequest &request, HTMLForm & params);
 
     /// Call after initialization before using system logs. Call for global context.
     void initializeSystemLogs();