提交 b82eee52 编写于 作者: Z zhang2014

ISSUES-5436 fix review suggestions & add some '?' re2 test

上级 8105a9bb
......@@ -12,7 +12,7 @@ namespace DB
namespace ErrorCodes
{
extern const int SYNTAX_ERROR;
extern const int UNKNOW_HTTP_HANDLER_TYPE;
extern const int UNKNOWN_HTTP_HANDLER_TYPE;
extern const int EMPTY_HTTP_HANDLER_IN_CONFIG;
}
......@@ -99,13 +99,13 @@ HTTPHandlerMatcher createRootHandlerMatcher(IServer &, const String &);
HTTPHandlerMatcher createPingHandlerMatcher(IServer &, const String &);
HTTPHandlerMatcher createDynamicQueryHandlerMatcher(IServer &, const String &);
HTTPHandlerMatcher createReplicasStatusHandlerMatcher(IServer &, const String &);
HTTPHandlerMatcher createPredefineQueryHandlerMatcher(IServer &, const String &);
HTTPHandlerMatcher createPredefinedQueryHandlerMatcher(IServer &, const String &);
HTTPHandlerCreator createRootHandlerCreator(IServer &, const String &);
HTTPHandlerCreator createPingHandlerCreator(IServer &, const String &);
HTTPHandlerCreator createDynamicQueryHandlerCreator(IServer &, const String &);
HTTPHandlerCreator createReplicasStatusHandlerCreator(IServer &, const String &);
HTTPHandlerCreator createPredefineQueryHandlerCreator(IServer &, const String &);
HTTPHandlerCreator createPredefinedQueryHandlerCreator(IServer &, const String &);
void HTTPHandlerFactory::updateHTTPHandlersCreator(Poco::Util::AbstractConfiguration & configuration, const String & key)
{
......@@ -126,14 +126,14 @@ void HTTPHandlerFactory::updateHTTPHandlersCreator(Poco::Util::AbstractConfigura
handlers_creator.push_back({createPingHandlerMatcher(server, handler_key), createPingHandlerCreator(server, handler_key)});
else if (startsWith(http_handler_type_name, "dynamic_query_handler"))
handlers_creator.push_back({createDynamicQueryHandlerMatcher(server, handler_key), createDynamicQueryHandlerCreator(server, handler_key)});
else if (startsWith(http_handler_type_name, "predefine_query_handler"))
handlers_creator.push_back({createPredefineQueryHandlerMatcher(server, handler_key), createPredefineQueryHandlerCreator(server, handler_key)});
else if (startsWith(http_handler_type_name, "predefined_query_handler"))
handlers_creator.push_back({createPredefinedQueryHandlerMatcher(server, handler_key), createPredefinedQueryHandlerCreator(server, handler_key)});
else if (startsWith(http_handler_type_name, "replicas_status_handler"))
handlers_creator.push_back({createReplicasStatusHandlerMatcher(server, handler_key), createReplicasStatusHandlerCreator(server, handler_key)});
else if (http_handler_type_name == "no_handler_description")
no_handler_description = configuration.getString(key + ".no_handler_description");
else
throw Exception("Unknown HTTPHandler type name: " + http_handler_type_name, ErrorCodes::UNKNOW_HTTP_HANDLER_TYPE);
throw Exception("Unknown HTTPHandler type name: " + http_handler_type_name, ErrorCodes::UNKNOWN_HTTP_HANDLER_TYPE);
}
}
......
......@@ -56,7 +56,7 @@ public:
/// Settings can be overridden in the query.
/// Some parameters (database, default_format, everything used in the code above) do not
/// belong to the Settings class.
becomeReadonlyIfNeed(request);
becomeReadonlyIfNeeded(request);
changeSettingsFromParams(params, reservedParamSuffixesFilter(is_multipart_data));
if (is_multipart_data || settings_may_in_post)
......@@ -82,7 +82,7 @@ private:
/// In theory if initially readonly = 0, the client can change any setting and then set readonly
/// to some other value.
/// Only readonly queries are allowed for HTTP GET requests.
void becomeReadonlyIfNeed(Poco::Net::HTTPServerRequest & request)
void becomeReadonlyIfNeeded(Poco::Net::HTTPServerRequest & request)
{
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET)
{
......
......@@ -174,6 +174,6 @@ HTTPResponseBufferPtr HTTPQueryRequestHandler<QueryParamExtractor>::createRespon
template class HTTPQueryRequestHandler<ExtractorDynamicQueryParameters>;
template class HTTPQueryRequestHandler<ExtractorPredefineQueryParameters>;
template class HTTPQueryRequestHandler<ExtractorPredefinedQueryParameters>;
}
......@@ -9,12 +9,12 @@ namespace DB
namespace ErrorCodes
{
extern const int EMPTY_PREDEFINE_QUERY;
extern const int EMPTY_PREDEFINED_QUERY;
extern const int CANNOT_COMPILE_REGEXP;
extern const int UNKNOWN_QUERY_PARAMETER;
extern const int DUPLICATE_CAPTURE_QUERY_PARAM;
extern const int ILLEGAL_HTTP_HANDLER_PARAM_NAME;
extern const int TOO_MANY_INSERT_QUERY_WITH_PREDEFINE_QUERY;
extern const int TOO_MANY_INSERT_QUERY_WITH_PREDEFINED_QUERY;
}
ExtractorDynamicQueryParameters::ExtractorDynamicQueryParameters(
......@@ -62,6 +62,9 @@ ExtractorDynamicQueryParameters::ExtractorDynamicQueryParameters(
template <bool remove_prefix_for_param = false>
void extractParamWithRegex(Context & context, const RegexRule & regex, const std::map<String, int> & extract_params, const String & value)
{
if (value.empty())
return;
int num_captures = regex->NumberOfCapturingGroups() + 1;
re2_st::StringPiece matches[num_captures];
......@@ -71,16 +74,19 @@ void extractParamWithRegex(Context & context, const RegexRule & regex, const std
{
for (const auto & [capturing_name, capturing_index] : extract_params)
{
String param_name = capturing_name;
const auto & capturing_value = matches[capturing_index];
if constexpr (remove_prefix_for_param)
if (capturing_value.data())
{
const static size_t prefix_size = strlen("param_");
param_name = capturing_name.substr(prefix_size);
}
String param_name = capturing_name;
if constexpr (remove_prefix_for_param)
{
const static size_t prefix_size = strlen("param_");
param_name = capturing_name.substr(prefix_size);
}
context.setQueryParameter(param_name, String(capturing_value.data(), capturing_value.size()));
context.setQueryParameter(param_name, String(capturing_value.data(), capturing_value.size()));
}
}
}
}
......@@ -93,7 +99,7 @@ ExtractRes ExtractorDynamicQueryParameters::extract(Context & context, Poco::Net
if (!extract_from_headers.empty())
for (const auto & [header_name, extract_params] : extract_from_headers)
extractParamWithRegex<true>(context, headers_regex.at(header_name), extract_params, request.get(header_name));
extractParamWithRegex<true>(context, headers_regex.at(header_name), extract_params, request.get(header_name, ""));
String extracted_query_from_params;
const static size_t prefix_size = strlen("param_");
......@@ -112,7 +118,7 @@ ExtractRes ExtractorDynamicQueryParameters::extract(Context & context, Poco::Net
return {{extracted_query_from_params, true}};
}
ExtractorPredefineQueryParameters::ExtractorPredefineQueryParameters(
ExtractorPredefinedQueryParameters::ExtractorPredefinedQueryParameters(
Poco::Util::AbstractConfiguration & configuration, const String & key, const RegexRule & url_regex_, const HeadersRegexRule & headers_regex_)
: url_regex(url_regex_), headers_regex(headers_regex_)
{
......@@ -120,7 +126,7 @@ ExtractorPredefineQueryParameters::ExtractorPredefineQueryParameters(
configuration.keys(key + ".queries", queries_key);
if (queries_key.empty())
throw Exception("There must be at least one predefined query in the predefined HTTPHandler.", ErrorCodes::EMPTY_PREDEFINE_QUERY);
throw Exception("There must be at least one predefined query in the predefined HTTPHandler.", ErrorCodes::EMPTY_PREDEFINED_QUERY);
for (const auto & query_key : queries_key)
{
......@@ -136,7 +142,7 @@ ExtractorPredefineQueryParameters::ExtractorPredefineQueryParameters(
bool is_insert_query = extract_query_ast->as<ASTInsertQuery>();
if (has_insert_query && is_insert_query)
throw Exception("Too many insert queries in predefined queries.", ErrorCodes::TOO_MANY_INSERT_QUERY_WITH_PREDEFINE_QUERY);
throw Exception("Too many insert queries in predefined queries.", ErrorCodes::TOO_MANY_INSERT_QUERY_WITH_PREDEFINED_QUERY);
has_insert_query |= is_insert_query;
predefine_queries.push_back({predefine_query, is_insert_query});
......@@ -186,14 +192,14 @@ ExtractorPredefineQueryParameters::ExtractorPredefineQueryParameters(
}
}
ExtractRes ExtractorPredefineQueryParameters::extract(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params)
ExtractRes ExtractorPredefinedQueryParameters::extract(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params)
{
if (!extract_from_url.empty())
extractParamWithRegex<false>(context, url_regex, extract_from_url, Poco::URI{request.getURI()}.getPath());
if (!extract_from_headers.empty())
for (const auto & [header_name, extract_params] : extract_from_headers)
extractParamWithRegex<false>(context, headers_regex.at(header_name), extract_params, request.get(header_name));
extractParamWithRegex<false>(context, headers_regex.at(header_name), extract_params, request.get(header_name, ""));
for (const auto & param : params)
if (queries_names.count(param.first))
......@@ -277,6 +283,9 @@ HTTPHandlerMatcher HTTPQueryRequestHandlerMatcherAndCreator::createHandlerMatche
{
for (const auto & [header_name, header_rule] : headers_rule)
{
if (!request.has(header_name))
return false;
const String & header_value = request.get(header_name);
if (header_value.size() != findFirstMissingMatchPos(*header_rule, header_value))
return false;
......@@ -294,7 +303,7 @@ HTTPHandlerMatcher createDynamicQueryHandlerMatcher(IServer & server, const Stri
}
HTTPHandlerMatcher createPredefineQueryHandlerMatcher(IServer & server, const String & key)
HTTPHandlerMatcher createPredefinedQueryHandlerMatcher(IServer & server, const String & key)
{
return HTTPQueryRequestHandlerMatcherAndCreator::invokeWithParsedRegexRule(server.config(), key,
HTTPQueryRequestHandlerMatcherAndCreator::createHandlerMatcher);
......@@ -314,16 +323,16 @@ HTTPHandlerCreator createDynamicQueryHandlerCreator(IServer & server, const Stri
});
}
HTTPHandlerCreator createPredefineQueryHandlerCreator(IServer & server, const String & key)
HTTPHandlerCreator createPredefinedQueryHandlerCreator(IServer & server, const String & key)
{
return HTTPQueryRequestHandlerMatcherAndCreator::invokeWithParsedRegexRule(
server.config(), key, [&](const String &, const RegexRule & url_rule, const HeadersRegexRule & headers_rule)
{
const auto & extract = std::make_shared<ExtractorPredefineQueryParameters>(server.config(), key, url_rule, headers_rule);
const auto & extract = std::make_shared<ExtractorPredefinedQueryParameters>(server.config(), key, url_rule, headers_rule);
return [&, query_extract = extract]()
{
return new HTTPQueryRequestHandler<ExtractorPredefineQueryParameters>(server, *query_extract);
return new HTTPQueryRequestHandler<ExtractorPredefinedQueryParameters>(server, *query_extract);
};
});
}
......
......@@ -48,10 +48,10 @@ private:
std::map<String, std::map<String, int>> extract_from_headers;
};
class ExtractorPredefineQueryParameters
class ExtractorPredefinedQueryParameters
{
public:
ExtractorPredefineQueryParameters(
ExtractorPredefinedQueryParameters(
Poco::Util::AbstractConfiguration & configuration, const String & key,
const RegexRule & url_regex_, const HeadersRegexRule & headers_regex_
);
......
......@@ -7,6 +7,7 @@
namespace DB
{
/// Manage the lifetime of the session context.
struct HTTPSessionContextHolder
{
~HTTPSessionContextHolder();
......
......@@ -4,14 +4,14 @@
<http_handlers>
<root_handler/>
<ping_handler>/ping_test</ping_handler>
<predefine_query_handler>
<predefined_query_handler>
<method>POST</method>
<url>/test_one_handler_with_insert_and_select</url>
<queries>
<query>INSERT INTO test.test VALUES(1)</query>
<query>SELECT COUNT() FROM test.test WHERE id = {id:UInt8}</query>
</queries>
</predefine_query_handler>
</predefined_query_handler>
<no_handler_description>test not found</no_handler_description>
</http_handlers>
......
......@@ -35,8 +35,8 @@ def test_dynamic_query_handler_with_insert_and_select():
assert cluster.instance.http_request('query_data_from_test?max_threads=1&test_select_query_param=' + select_data_query, method='GET') == '1\n2\n3\n4\n'
def test_predefine_query_handler_with_insert_and_select():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "predefine_insert_and_select", "test_insert_and_select_predefine")) as cluster:
def test_predefined_query_handler_with_insert_and_select():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "predefined_insert_and_select", "test_insert_and_select_predefined")) as cluster:
assert cluster.instance.http_request('create_test_table?max_threads=1', method='PUT') == ''
assert cluster.instance.http_request('insert_data_to_test?max_threads=1', method='POST', data='(1)(2)(3)(4)') == ''
assert cluster.instance.http_request('query_data_from_test?max_threads=1', method='GET') == '1\n2\n3\n4\n'
......@@ -54,18 +54,25 @@ def test_dynamic_query_handler_with_params_and_settings():
assert 'Syntax error' in cluster.instance.http_request('post_query_params_and_settings?post_query_param=' + test_query + '&' + query_param, method='POST', data=settings)
assert cluster.instance.http_request('get_query_params_and_settings?get_query_param=' + quoted_test_query + '&' + query_param + '&' + settings) == '1\n2\n'
assert cluster.instance.http_request('query_param_with_url/123/max_threads?query_param=' + quoted_test_query + '&' + settings + '&param_name_2=max_alter_threads') == '1\n2\n'
assert cluster.instance.http_request('query_param_with_url/123/max_threads/max_alter_threads?query_param=' + quoted_test_query + '&' + settings) == '1\n2\n'
assert '`name_2` is not set' in cluster.instance.http_request('query_param_with_url/123/max_threads?query_param=' + quoted_test_query + '&' + settings)
assert 'Duplicate name' in cluster.instance.http_request('query_param_with_url/123/max_threads_dump/max_alter_threads_dump?query_param=' + quoted_test_query + '&' + query_param + '&' + settings)
assert cluster.instance.http_request('test_match_headers?query_param=' + quoted_test_query + '&' + settings, headers={'XXX': 'TEST_HEADER_VALUE', 'PARAMS_XXX': 'max_threads/max_alter_threads'}) == '1\n2\n'
assert cluster.instance.http_request('test_match_headers?query_param=' + quoted_test_query + '&' + settings + '&param_name_2=max_alter_threads', headers={'XXX': 'TEST_HEADER_VALUE', 'PARAMS_XXX': 'max_threads'}) == '1\n2\n'
assert '`name_2` is not set' in cluster.instance.http_request('test_match_headers?query_param=' + quoted_test_query + '&' + settings, headers={'XXX': 'TEST_HEADER_VALUE', 'PARAMS_XXX': 'max_threads'})
assert 'There is no handle /test_match_headers' in cluster.instance.http_request('test_match_headers?query_param=' + quoted_test_query + '&' + settings)
def test_predefine_query_handler_with_params_and_settings():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "predefine_params_and_settings", "test_param_and_settings_predefine")) as cluster:
def test_predefined_query_handler_with_params_and_settings():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "predefined_params_and_settings", "test_param_and_settings_predefined")) as cluster:
settings = 'max_threads=1&max_alter_threads=2'
query_param = 'name_1=max_threads&name_2=max_alter_threads'
assert cluster.instance.http_request('get_query_params_and_settings?' + query_param + '&' + settings, method='GET') == '1\nmax_alter_threads\t2\n'
assert cluster.instance.http_request('query_param_with_url/123/max_threads/max_alter_threads?' + settings) == '1\nmax_alter_threads\t2\n'
assert cluster.instance.http_request('query_param_with_url/123/max_threads?' + settings + '&name_2=max_alter_threads') == '1\nmax_alter_threads\t2\n'
assert '`name_2` is not set' in cluster.instance.http_request('query_param_with_url/123/max_threads?' + settings)
assert 'Duplicate name' in cluster.instance.http_request('query_param_with_url/123/max_threads_dump/max_alter_threads_dump?' + query_param + '&' + settings)
assert cluster.instance.http_request('post_query_params_and_settings?' + query_param, method='POST', data=settings) == '1\nmax_alter_threads\t2\n'
......@@ -73,6 +80,9 @@ def test_predefine_query_handler_with_params_and_settings():
assert cluster.instance.http_request('post_query_params_and_settings?' + query_param + '&' + settings, method='POST') == '1\nmax_alter_threads\t2\n'
assert cluster.instance.http_request('post_query_params_and_settings', method='POST', data=query_param + '&' + settings) == '1\nmax_alter_threads\t2\n'
assert cluster.instance.http_request('test_match_headers?' + settings, headers={'XXX': 'TEST_HEADER_VALUE', 'PARAMS_XXX': 'max_threads/max_alter_threads'}) == '1\nmax_alter_threads\t2\n'
assert cluster.instance.http_request('test_match_headers?' + settings + '&name_2=max_alter_threads', headers={'XXX': 'TEST_HEADER_VALUE', 'PARAMS_XXX': 'max_threads'}) == '1\nmax_alter_threads\t2\n'
assert '`name_2` is not set' in cluster.instance.http_request('test_match_headers?' + settings, headers={'XXX': 'TEST_HEADER_VALUE', 'PARAMS_XXX': 'max_threads'})
assert 'There is no handle /test_match_headers' in cluster.instance.http_request('test_match_headers?' + settings)
def test_other_configs():
......
......@@ -2,25 +2,25 @@
<yandex>
<http_handlers>
<predefine_query_handler>
<predefined_query_handler>
<method>PUT</method>
<url>/create_test_table</url>
<queries>
<query>CREATE DATABASE test</query>
<query>CREATE TABLE test.test (id UInt8) Engine = Memory</query>
</queries>
</predefine_query_handler>
</predefined_query_handler>
<predefine_query_handler>
<predefined_query_handler>
<method>POST</method>
<url>/insert_data_to_test</url>
<queries><query>INSERT INTO test.test VALUES</query></queries>
</predefine_query_handler>
</predefined_query_handler>
<predefine_query_handler>
<predefined_query_handler>
<method>GET</method>
<url>/query_data_from_test</url>
<queries><query>SELECT * FROM test.test ORDER BY id</query></queries>
</predefine_query_handler>
</predefined_query_handler>
</http_handlers>
</yandex>
......@@ -16,14 +16,14 @@
<dynamic_query_handler>
<method>GET</method>
<url><![CDATA[/query_param_with_url/\w+/(?P<param_name_1>.+)/(?P<param_name_2>.+)]]></url>
<url><![CDATA[/query_param_with_url/\w+/(?P<param_name_1>[^/]+)(/(?P<param_name_2>[^/]+))?]]></url>
<query_param_name>query_param</query_param_name>
</dynamic_query_handler>
<dynamic_query_handler>
<headers>
<XXX>TEST_HEADER_VALUE</XXX>
<PARAMS_XXX><![CDATA[(?P<param_name_1>.+)/(?P<param_name_2>.+)]]></PARAMS_XXX>
<PARAMS_XXX><![CDATA[(?P<param_name_1>[^/]+)(/(?P<param_name_2>[^/]+))?]]></PARAMS_XXX>
</headers>
<query_param_name>query_param</query_param_name>
</dynamic_query_handler>
......
......@@ -2,42 +2,42 @@
<yandex>
<http_handlers>
<predefine_query_handler>
<predefined_query_handler>
<method>POST</method>
<url>/post_query_params_and_settings</url>
<queries>
<query>SELECT value FROM system.settings WHERE name = {name_1:String}</query>
<query>SELECT name, value FROM system.settings WHERE name = {name_2:String}</query>
</queries>
</predefine_query_handler>
</predefined_query_handler>
<predefine_query_handler>
<predefined_query_handler>
<method>GET</method>
<url>/get_query_params_and_settings</url>
<queries>
<query>SELECT value FROM system.settings WHERE name = {name_1:String}</query>
<query>SELECT name, value FROM system.settings WHERE name = {name_2:String}</query>
</queries>
</predefine_query_handler>
</predefined_query_handler>
<predefine_query_handler>
<predefined_query_handler>
<method>GET</method>
<url><![CDATA[/query_param_with_url/\w+/(?P<name_1>.+)/(?P<name_2>.+)]]></url>
<url><![CDATA[/query_param_with_url/\w+/(?P<name_1>[^/]+)(/(?P<name_2>[^/]+))?]]></url>
<queries>
<query>SELECT value FROM system.settings WHERE name = {name_1:String}</query>
<query>SELECT name, value FROM system.settings WHERE name = {name_2:String}</query>
</queries>
</predefine_query_handler>
</predefined_query_handler>
<predefine_query_handler>
<predefined_query_handler>
<headers>
<XXX>TEST_HEADER_VALUE</XXX>
<PARAMS_XXX><![CDATA[(?P<name_1>.+)/(?P<name_2>.+)]]></PARAMS_XXX>
<PARAMS_XXX><![CDATA[(?P<name_1>[^/]+)(/(?P<name_2>[^/]+))?]]></PARAMS_XXX>
</headers>
<queries>
<query>SELECT value FROM system.settings WHERE name = {name_1:String}</query>
<query>SELECT name, value FROM system.settings WHERE name = {name_2:String}</query>
</queries>
</predefine_query_handler>
</predefined_query_handler>
</http_handlers>
</yandex>
......@@ -537,8 +537,8 @@
url - RE2 compatible regular expression (optional)
method - HTTP method(optional)
headers - HTTP Header(optional)
queries - predefine queries (mandatory)
<predefine_query_handler>
queries - predefined queries (mandatory)
<predefined_query_handler>
<url>/test_simple_predefine</url>
<method>GET</method>
<headers> <X-ClickHouse-User>default</X-ClickHouse-User></headers>
......@@ -546,7 +546,7 @@
<query>SELECT 1, {query_prepared_param_1:String}</query>
<query>SELECT 1, {query_prepared_param_2:String}</query>
</queries>
</predefine_query_handler>
</predefined_query_handler>
-->
</http_handlers>
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册