Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
ea734b6b
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ea734b6b
编写于
7月 30, 2016
作者:
A
Alexey Milovidov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Removed compatibility layer with OLAPServer [#METR-2944].
上级
c6da35b7
变更
14
展开全部
隐藏空白更改
内联
并排
Showing
14 changed file
with
12 addition
and
2328 deletion
+12
-2328
dbms/CMakeLists.txt
dbms/CMakeLists.txt
+0
-4
dbms/include/DB/Interpreters/Context.h
dbms/include/DB/Interpreters/Context.h
+0
-1
dbms/src/Server/CMakeLists.txt
dbms/src/Server/CMakeLists.txt
+0
-3
dbms/src/Server/OLAPAttributesMetadata.h
dbms/src/Server/OLAPAttributesMetadata.h
+0
-805
dbms/src/Server/OLAPHTTPHandler.cpp
dbms/src/Server/OLAPHTTPHandler.cpp
+0
-138
dbms/src/Server/OLAPHTTPHandler.h
dbms/src/Server/OLAPHTTPHandler.h
+0
-32
dbms/src/Server/OLAPQueryConverter.cpp
dbms/src/Server/OLAPQueryConverter.cpp
+0
-739
dbms/src/Server/OLAPQueryConverter.h
dbms/src/Server/OLAPQueryConverter.h
+0
-87
dbms/src/Server/OLAPQueryParser.cpp
dbms/src/Server/OLAPQueryParser.cpp
+0
-341
dbms/src/Server/OLAPQueryParser.h
dbms/src/Server/OLAPQueryParser.h
+0
-135
dbms/src/Server/Server.cpp
dbms/src/Server/Server.cpp
+0
-23
dbms/src/Server/Server.h
dbms/src/Server/Server.h
+9
-14
dbms/src/Server/TCPHandler.cpp
dbms/src/Server/TCPHandler.cpp
+1
-4
libs/libcommon/include/common/DateLUTImpl.h
libs/libcommon/include/common/DateLUTImpl.h
+2
-2
未找到文件。
dbms/CMakeLists.txt
浏览文件 @
ea734b6b
...
@@ -16,14 +16,10 @@ else()
...
@@ -16,14 +16,10 @@ else()
endif
()
endif
()
add_library
(
dbms
add_library
(
dbms
src/Server/OLAPAttributesMetadata.h
src/Server/InterserverIOHTTPHandler.h
src/Server/InterserverIOHTTPHandler.h
src/Server/OLAPHTTPHandler.h
src/Server/OLAPQueryConverter.h
src/Server/Server.h
src/Server/Server.h
src/Server/TCPHandler.h
src/Server/TCPHandler.h
src/Server/HTTPHandler.h
src/Server/HTTPHandler.h
src/Server/OLAPQueryParser.h
src/Server/MetricsTransmitter.h
src/Server/MetricsTransmitter.h
src/Server/UsersConfigReloader.h
src/Server/UsersConfigReloader.h
src/Server/StatusFile.h
src/Server/StatusFile.h
...
...
dbms/include/DB/Interpreters/Context.h
浏览文件 @
ea734b6b
...
@@ -67,7 +67,6 @@ public:
...
@@ -67,7 +67,6 @@ public:
{
{
TCP
=
1
,
TCP
=
1
,
HTTP
=
2
,
HTTP
=
2
,
OLAP_HTTP
=
3
,
};
};
enum
class
HTTPMethod
enum
class
HTTPMethod
...
...
dbms/src/Server/CMakeLists.txt
浏览文件 @
ea734b6b
...
@@ -3,9 +3,6 @@ add_executable(clickhouse-server
...
@@ -3,9 +3,6 @@ add_executable(clickhouse-server
HTTPHandler.cpp
HTTPHandler.cpp
TCPHandler.cpp
TCPHandler.cpp
InterserverIOHTTPHandler.cpp
InterserverIOHTTPHandler.cpp
OLAPHTTPHandler.cpp
OLAPQueryParser.cpp
OLAPQueryConverter.cpp
MetricsTransmitter.cpp
MetricsTransmitter.cpp
UsersConfigReloader.cpp
UsersConfigReloader.cpp
StatusFile.cpp
StatusFile.cpp
...
...
dbms/src/Server/OLAPAttributesMetadata.h
已删除
100644 → 0
浏览文件 @
c6da35b7
此差异已折叠。
点击以展开。
dbms/src/Server/OLAPHTTPHandler.cpp
已删除
100644 → 0
浏览文件 @
c6da35b7
#include <Poco/Net/HTTPBasicCredentials.h>
#include <DB/Interpreters/executeQuery.h>
#include <DB/Interpreters/Quota.h>
#include <DB/IO/WriteBufferFromHTTPServerResponse.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ReadBufferFromIStream.h>
#include <DB/IO/ReadBufferFromString.h>
#include "OLAPQueryParser.h"
#include "OLAPQueryConverter.h"
#include "OLAPHTTPHandler.h"
#include <DB/Common/Stopwatch.h>
#include <iomanip>
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
POCO_EXCEPTION
;
extern
const
int
STD_EXCEPTION
;
extern
const
int
UNKNOWN_EXCEPTION
;
}
void
OLAPHTTPHandler
::
processQuery
(
Poco
::
Net
::
HTTPServerRequest
&
request
,
Poco
::
Net
::
HTTPServerResponse
&
response
)
{
HTMLForm
params
(
request
);
std
::
ostringstream
request_ostream
;
request_ostream
<<
request
.
stream
().
rdbuf
();
std
::
string
request_string
=
request_ostream
.
str
();
LOG_TRACE
(
log
,
"Request URI: "
<<
request
.
getURI
());
LOG_TRACE
(
log
,
"Request body: "
<<
request_string
);
std
::
istringstream
request_istream
(
request_string
);
BlockInputStreamPtr
query_plan
;
/// Имя пользователя и пароль могут быть заданы как в параметрах URL, так и с помощью HTTP Basic authentification (и то, и другое не секъюрно).
std
::
string
user
=
params
.
get
(
"user"
,
"default"
);
std
::
string
password
=
params
.
get
(
"password"
,
""
);
std
::
string
quota_key
=
params
.
get
(
"quota_key"
,
""
);
if
(
request
.
hasCredentials
())
{
Poco
::
Net
::
HTTPBasicCredentials
credentials
(
request
);
user
=
credentials
.
getUsername
();
password
=
credentials
.
getPassword
();
}
Context
context
=
*
server
.
global_context
;
context
.
setGlobalContext
(
*
server
.
global_context
);
context
.
setSetting
(
"profile"
,
profile
);
context
.
setUser
(
user
,
password
,
request
.
clientAddress
().
host
(),
quota_key
);
context
.
setInterface
(
Context
::
Interface
::
OLAP_HTTP
);
context
.
setHTTPMethod
(
Context
::
HTTPMethod
::
POST
);
OLAP
::
QueryParseResult
olap_query
=
server
.
olap_parser
->
parse
(
request_istream
);
std
::
string
clickhouse_query
;
server
.
olap_converter
->
OLAPServerQueryToClickHouse
(
olap_query
,
context
,
clickhouse_query
);
LOG_TRACE
(
log
,
"Converted query: "
<<
clickhouse_query
);
ReadBufferFromString
in
(
clickhouse_query
);
WriteBufferFromHTTPServerResponse
out
(
response
);
Stopwatch
watch
;
executeQuery
(
in
,
out
,
context
,
query_plan
,
[
&
response
]
(
const
String
&
content_type
)
{
response
.
setContentType
(
content_type
);
});
watch
.
stop
();
/// Если не было эксепшена и данные ещё не отправлены - отправляются HTTP заголовки с кодом 200.
out
.
finalize
();
}
void
OLAPHTTPHandler
::
handleRequest
(
Poco
::
Net
::
HTTPServerRequest
&
request
,
Poco
::
Net
::
HTTPServerResponse
&
response
)
{
/// Для того, чтобы работал keep-alive.
if
(
request
.
getVersion
()
==
Poco
::
Net
::
HTTPServerRequest
::
HTTP_1_1
)
response
.
setChunkedTransferEncoding
(
true
);
try
{
processQuery
(
request
,
response
);
LOG_INFO
(
log
,
"Done processing query"
);
}
catch
(
Exception
&
e
)
{
response
.
setStatusAndReason
(
Poco
::
Net
::
HTTPResponse
::
HTTP_INTERNAL_SERVER_ERROR
);
std
::
stringstream
s
;
s
<<
"Code: "
<<
e
.
code
()
<<
", e.displayText() = "
<<
e
.
displayText
()
<<
", e.what() = "
<<
e
.
what
();
if
(
!
response
.
sent
())
response
.
send
()
<<
s
.
str
()
<<
std
::
endl
;
LOG_ERROR
(
log
,
s
.
str
());
}
catch
(
Poco
::
Exception
&
e
)
{
response
.
setStatusAndReason
(
Poco
::
Net
::
HTTPResponse
::
HTTP_INTERNAL_SERVER_ERROR
);
std
::
stringstream
s
;
s
<<
"Code: "
<<
ErrorCodes
::
POCO_EXCEPTION
<<
", e.code() = "
<<
e
.
code
()
<<
", e.displayText() = "
<<
e
.
displayText
()
<<
", e.what() = "
<<
e
.
what
();
if
(
!
response
.
sent
())
response
.
send
()
<<
s
.
str
()
<<
std
::
endl
;
LOG_ERROR
(
log
,
s
.
str
());
}
catch
(
std
::
exception
&
e
)
{
response
.
setStatusAndReason
(
Poco
::
Net
::
HTTPResponse
::
HTTP_INTERNAL_SERVER_ERROR
);
std
::
stringstream
s
;
s
<<
"Code: "
<<
ErrorCodes
::
STD_EXCEPTION
<<
". "
<<
e
.
what
();
if
(
!
response
.
sent
())
response
.
send
()
<<
s
.
str
()
<<
std
::
endl
;
LOG_ERROR
(
log
,
s
.
str
());
}
catch
(...)
{
response
.
setStatusAndReason
(
Poco
::
Net
::
HTTPResponse
::
HTTP_INTERNAL_SERVER_ERROR
);
std
::
stringstream
s
;
s
<<
"Code: "
<<
ErrorCodes
::
UNKNOWN_EXCEPTION
<<
". Unknown exception."
;
if
(
!
response
.
sent
())
response
.
send
()
<<
s
.
str
()
<<
std
::
endl
;
LOG_ERROR
(
log
,
s
.
str
());
}
}
}
dbms/src/Server/OLAPHTTPHandler.h
已删除
100644 → 0
浏览文件 @
c6da35b7
#pragma once
#include "Server.h"
#include <DB/Common/CurrentMetrics.h>
namespace
DB
{
/// Обработчик http-запросов в формате OLAP-server.
class
OLAPHTTPHandler
:
public
Poco
::
Net
::
HTTPRequestHandler
{
public:
OLAPHTTPHandler
(
Server
&
server_
)
:
server
(
server_
),
log
(
&
Logger
::
get
(
"OLAPHTTPHandler"
)),
profile
(
Poco
::
Util
::
Application
::
instance
().
config
().
getString
(
"olap_compatibility.profile"
))
{
}
void
handleRequest
(
Poco
::
Net
::
HTTPServerRequest
&
request
,
Poco
::
Net
::
HTTPServerResponse
&
response
);
private:
Server
&
server
;
Logger
*
log
;
const
String
profile
;
CurrentMetrics
::
Increment
metric_increment
{
CurrentMetrics
::
HTTPConnection
};
void
processQuery
(
Poco
::
Net
::
HTTPServerRequest
&
request
,
Poco
::
Net
::
HTTPServerResponse
&
response
);
};
}
dbms/src/Server/OLAPQueryConverter.cpp
已删除
100644 → 0
浏览文件 @
c6da35b7
此差异已折叠。
点击以展开。
dbms/src/Server/OLAPQueryConverter.h
已删除
100644 → 0
浏览文件 @
c6da35b7
#pragma once
#include "OLAPQueryParser.h"
#include <DB/Interpreters/Context.h>
#include <Poco/Util/AbstractConfiguration.h>
#include "OLAPAttributesMetadata.h"
namespace
DB
{
namespace
OLAP
{
/// Конвертирует распаршенный XML-запрос в формате OLAP-server в SQL-подобный запрос для clickhouse.
class
QueryConverter
{
public:
QueryConverter
(
Poco
::
Util
::
AbstractConfiguration
&
config
);
QueryConverter
(
const
String
&
table_for_single_counter
,
const
String
&
table_for_all_counters
);
/// Получает из запроса в формате OLAP-server запрос и настройки для clickhouse.
void
OLAPServerQueryToClickHouse
(
const
QueryParseResult
&
query
,
Context
&
inout_context
,
std
::
string
&
out_query
)
const
;
private:
/// Значение атрибута, подходящее для вывода в ответ и для группировки по нему.
std
::
string
convertAttributeFormatted
(
const
std
::
string
&
attribute
,
unsigned
parameter
,
const
std
::
string
&
regions_point_of_view_formatted
)
const
;
/// Числовое значение атрибута, подходящее для подстановки в условия, агрегатные функции и ключи сортировки.
std
::
string
convertAttributeNumeric
(
const
std
::
string
&
attribute
,
unsigned
parameter
,
const
std
::
string
&
regions_point_of_view_formatted
)
const
;
/// <aggregates><aggregate> => SELECT x
std
::
string
convertAggregateFunction
(
const
std
::
string
&
attribute
,
unsigned
parameter
,
const
std
::
string
&
function
,
const
QueryParseResult
&
query
,
const
std
::
string
&
regions_point_of_view_formatted
)
const
;
/// <where><condition><rhs> => SELECT ... where F(A, x)
std
::
string
convertConstant
(
const
std
::
string
&
attribute
,
const
std
::
string
&
value
)
const
;
/// <where><condition> => SELECT ... WHERE x
std
::
string
convertCondition
(
const
std
::
string
&
attribute
,
unsigned
parameter
,
const
std
::
string
&
relation
,
const
std
::
string
&
rhs
,
const
std
::
string
&
regions_point_of_view_formatted
)
const
;
/// ASC или DESC
std
::
string
convertSortDirection
(
const
std
::
string
&
direction
)
const
;
/// <dates> => SELECT ... WHERE x
std
::
string
convertDateRange
(
time_t
date_first
,
time_t
date_last
)
const
;
/// <counter_id> => SELECT ... WHERE x
std
::
string
convertCounterID
(
CounterID_t
CounterID
)
const
;
std
::
string
getTableName
(
CounterID_t
CounterID
,
bool
local
)
const
;
std
::
string
getHavingSection
()
const
;
void
fillFormattedAttributeMap
();
void
fillNumericAttributeMap
();
void
fillFormattingAggregatedAttributeMap
();
std
::
string
table_for_single_counter
;
std
::
string
table_for_all_counters
;
/// Форматная строка для convertAttributeNumeric. Есть для всех атрибутов.
std
::
map
<
std
::
string
,
std
::
string
>
numeric_attribute_map
;
/// Форматная строка для получения выводимого значения из агрегированного числового значения.
std
::
map
<
std
::
string
,
std
::
string
>
formatting_aggregated_attribute_map
;
/// Форматная строка для convertAttributeFormatted.
std
::
map
<
std
::
string
,
std
::
string
>
formatted_attribute_map
;
/// Список атрибутов-регионов, для которых нужна передача параметра regions_point_of_view.
std
::
set
<
std
::
string
>
regions_attributes_set
=
{
"RegionCity"
,
"RegionArea"
,
"RegionCountry"
,
};
/// Парсеры значений атрибутов.
AttributeMetadatas
attribute_metadatas
;
};
}
}
dbms/src/Server/OLAPQueryParser.cpp
已删除
100644 → 0
浏览文件 @
c6da35b7
#include <common/DateLUT.h>
#include <Poco/DateTimeParser.h>
#include <Poco/AutoPtr.h>
#include "OLAPQueryParser.h"
#include <DB/Common/Exception.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
NOT_FOUND_NODE
;
extern
const
int
FOUND_MORE_THAN_ONE_NODE
;
extern
const
int
SYNTAX_ERROR
;
extern
const
int
UNKNOWN_FORMAT
;
extern
const
int
FIRST_DATE_IS_BIGGER_THAN_LAST_DATE
;
extern
const
int
UNKNOWN_OVERFLOW_MODE
;
extern
const
int
NOT_FOUND_FUNCTION_ELEMENT_FOR_AGGREGATE
;
extern
const
int
NOT_FOUND_RELATION_ELEMENT_FOR_CONDITION
;
extern
const
int
NOT_FOUND_RHS_ELEMENT_FOR_CONDITION
;
extern
const
int
NO_ATTRIBUTES_LISTED
;
extern
const
int
UNKNOWN_DIRECTION_OF_SORTING
;
extern
const
int
INDEX_OF_COLUMN_IN_SORT_CLAUSE_IS_OUT_OF_RANGE
;
}
namespace
OLAP
{
static
std
::
string
getValueOfOneTextElement
(
Poco
::
XML
::
Document
*
document
,
const
std
::
string
&
tag_name
)
{
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
node_list
=
document
->
getElementsByTagName
(
tag_name
);
if
(
0
==
node_list
->
length
())
throw
Exception
(
std
::
string
(
"Not found node "
)
+
tag_name
,
ErrorCodes
::
NOT_FOUND_NODE
);
else
if
(
1
!=
node_list
->
length
())
throw
Exception
(
std
::
string
(
"Found more than one node "
)
+
tag_name
,
ErrorCodes
::
FOUND_MORE_THAN_ONE_NODE
);
return
node_list
->
item
(
0
)
->
innerText
();
}
QueryParser
::
AttributeWithParameter
QueryParser
::
parseAttributeWithParameter
(
const
std
::
string
&
s
)
{
AttributeWithParameter
res
;
Poco
::
RegularExpression
::
MatchVec
matches
;
if
(
parse_attribute_with_parameter_regexp
.
match
(
s
,
0
,
matches
))
{
if
(
matches
.
size
()
==
3
)
{
res
.
first
=
s
.
substr
(
matches
[
1
].
offset
,
matches
[
1
].
length
);
res
.
second
=
DB
::
parse
<
unsigned
>
(
s
.
substr
(
matches
[
2
].
offset
,
matches
[
2
].
length
));
return
res
;
}
}
throw
Exception
(
std
::
string
(
"Invalid attribute syntax: "
)
+
s
,
ErrorCodes
::
SYNTAX_ERROR
);
}
QueryParseResult
QueryParser
::
parse
(
std
::
istream
&
s
)
{
QueryParseResult
result
;
Poco
::
XML
::
DOMParser
parser
;
Poco
::
XML
::
InputSource
source
(
s
);
result
.
max_result_size
=
0
;
result
.
max_execution_time
=
0
;
result
.
sample
=
1.0
;
result
.
query
=
parser
.
parse
(
&
source
);
std
::
string
format_element_name
(
"format"
);
std
::
string
CounterID_element_name
(
"counter_id"
);
std
::
string
date_first_element_name
(
"first"
);
std
::
string
date_last_element_name
(
"last"
);
result
.
format
=
FORMAT_XML
;
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
node_list
=
result
.
query
->
getElementsByTagName
(
format_element_name
);
if
(
node_list
->
length
()
>
1
)
throw
Exception
(
std
::
string
(
"Found more than one node "
)
+
format_element_name
,
ErrorCodes
::
FOUND_MORE_THAN_ONE_NODE
);
if
(
node_list
->
length
()
==
1
)
{
if
(
node_list
->
item
(
0
)
->
innerText
()
!=
"xml"
&&
node_list
->
item
(
0
)
->
innerText
()
!=
"tab"
&&
node_list
->
item
(
0
)
->
innerText
()
!=
"bin"
)
throw
Exception
(
std
::
string
(
"Unknown format: "
)
+
node_list
->
item
(
0
)
->
innerText
(),
ErrorCodes
::
UNKNOWN_FORMAT
);
result
.
format
=
(
node_list
->
item
(
0
)
->
innerText
()
==
"xml"
)
?
FORMAT_XML
:
((
node_list
->
item
(
0
)
->
innerText
()
==
"tab"
)
?
FORMAT_TAB
:
FORMAT_BIN
);
}
result
.
CounterID
=
0
;
if
(
result
.
query
->
getElementsByTagName
(
CounterID_element_name
)
->
length
()
>
0
)
result
.
CounterID
=
DB
::
parse
<
unsigned
>
(
getValueOfOneTextElement
(
result
.
query
,
CounterID_element_name
));
int
time_zone_diff
=
0
;
result
.
date_first
=
DateLUT
::
instance
().
toDate
(
Poco
::
DateTimeParser
::
parse
(
getValueOfOneTextElement
(
result
.
query
,
date_first_element_name
),
time_zone_diff
).
timestamp
().
epochTime
());
result
.
date_last
=
DateLUT
::
instance
().
toDate
(
Poco
::
DateTimeParser
::
parse
(
getValueOfOneTextElement
(
result
.
query
,
date_last_element_name
),
time_zone_diff
).
timestamp
().
epochTime
());
if
(
result
.
date_first
>
result
.
date_last
)
throw
Exception
(
"First date is bigger than last date."
,
ErrorCodes
::
FIRST_DATE_IS_BIGGER_THAN_LAST_DATE
);
const
auto
&
date_lut
=
DateLUT
::
instance
();
result
.
days
=
1
+
date_lut
.
toDayNum
(
result
.
date_last
)
-
date_lut
.
toDayNum
(
result
.
date_first
);
result
.
cut_date_last
=
false
;
result
.
cut_dates_for_goals
=
false
;
result
.
concurrency
=
0
;
result
.
max_threads_per_counter
=
0
;
result
.
limit
=
0
;
result
.
local
=
false
;
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
settings_nodes
=
result
.
query
->
getElementsByTagName
(
"settings"
);
if
(
settings_nodes
->
length
()
>
1
)
throw
Exception
(
std
::
string
(
"Found more than one node settings"
),
ErrorCodes
::
FOUND_MORE_THAN_ONE_NODE
);
if
(
settings_nodes
->
length
()
==
1
)
{
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
settings_child_nodes
=
settings_nodes
->
item
(
0
)
->
childNodes
();
for
(
unsigned
i
=
0
;
i
<
settings_child_nodes
->
length
();
i
++
)
{
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"max_result_size"
)
{
/// выставить дополнительное локальное ограничение на максимальный размер результата
result
.
max_result_size
=
DB
::
parse
<
unsigned
>
(
settings_child_nodes
->
item
(
i
)
->
innerText
());
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"max_execution_time"
)
{
/// выставить дополнительное локальное ограничение на максимальное время выполнения запроса
result
.
max_execution_time
=
DB
::
parse
<
unsigned
>
(
settings_child_nodes
->
item
(
i
)
->
innerText
());
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"cut_date_last"
)
{
/** обрезать запрошенный период до максимальной даты, за которую есть данные
* вместо того, чтобы сообщить об ошибке, если дата конца периода больше максимальной даты
*/
result
.
cut_date_last
=
true
;
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"cut_dates_for_goals"
)
{
/** если за какой-либо день не существовало цели, то пропускать этот день
*/
result
.
cut_dates_for_goals
=
true
;
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"overflow_mode"
)
{
/** определяет, что делать, если количество строк превышает max_result_size
*/
std
::
string
overflow_mode_str
=
settings_child_nodes
->
item
(
i
)
->
innerText
();
if
(
overflow_mode_str
!=
"throw"
&&
overflow_mode_str
!=
"break"
&&
overflow_mode_str
!=
"any"
)
throw
Exception
(
std
::
string
(
"Unknown overflow mode: "
)
+
overflow_mode_str
,
ErrorCodes
::
UNKNOWN_OVERFLOW_MODE
);
result
.
has_overflow_mode
=
true
;
result
.
overflow_mode
=
overflow_mode_str
==
"throw"
?
OVERFLOW_MODE_THROW
:
(
overflow_mode_str
==
"break"
?
OVERFLOW_MODE_BREAK
:
OVERFLOW_MODE_ANY
);
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"concurrency"
)
{
/// выставить количество потоков для обработки запроса
result
.
concurrency
=
DB
::
parse
<
unsigned
>
(
settings_child_nodes
->
item
(
i
)
->
innerText
());
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"max_threads_per_counter"
)
{
/** Выставить локальное ограничение на максимальное количество обрабатываемых запросов
* Оно может быть больше, чем ограничение по умолчанию.
*/
result
.
max_threads_per_counter
=
DB
::
parse
<
unsigned
>
(
settings_child_nodes
->
item
(
i
)
->
innerText
());
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"local"
)
{
result
.
local
=
true
;
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"sample"
)
{
result
.
sample
=
DB
::
parse
<
Float32
>
(
settings_child_nodes
->
item
(
i
)
->
innerText
());
if
(
result
.
sample
<=
0
||
result
.
sample
>
1.
)
throw
Exception
(
std
::
string
(
"Wrong sample = "
)
+
DB
::
toString
(
result
.
sample
)
+
". Sampling must be in range (0, 1]"
);
}
else
if
(
settings_child_nodes
->
item
(
i
)
->
nodeName
()
==
"regions_point_of_view"
)
{
result
.
regions_point_of_view
=
settings_child_nodes
->
item
(
i
)
->
innerText
();
}
}
}
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
limit_nodes
=
result
.
query
->
getElementsByTagName
(
"limit"
);
if
(
limit_nodes
->
length
()
>
1
)
throw
Exception
(
std
::
string
(
"Found more than one node limit"
),
ErrorCodes
::
FOUND_MORE_THAN_ONE_NODE
);
if
(
limit_nodes
->
length
()
==
1
)
result
.
limit
=
DB
::
parse
<
unsigned
>
(
limit_nodes
->
item
(
0
)
->
innerText
());
LOG_DEBUG
(
log
,
"CounterID: "
<<
result
.
CounterID
<<
", dates: "
<<
LocalDate
(
result
.
date_first
)
<<
" - "
<<
LocalDate
(
result
.
date_last
));
/// получаем список имён атрибутов
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
attributes
=
result
.
query
->
getElementsByTagName
(
"attribute"
);
for
(
unsigned
i
=
0
;
i
<
attributes
->
length
();
i
++
)
{
std
::
string
attribute_string
=
attributes
->
item
(
i
)
->
innerText
();
AttributeWithParameter
attr_with_param
;
std
::
string
&
attribute_name
=
attr_with_param
.
first
;
unsigned
&
attribute_param
=
attr_with_param
.
second
;
attribute_param
=
0
;
if
(
attribute_string
.
find
(
'('
)
!=
std
::
string
::
npos
)
attr_with_param
=
parseAttributeWithParameter
(
attribute_string
);
else
attribute_name
=
attribute_string
;
if
(
attributes
->
item
(
i
)
->
parentNode
()
->
nodeName
()
==
"keys"
)
{
QueryParseResult
::
KeyAttribute
key_attribute
;
key_attribute
.
attribute
=
attribute_name
;
key_attribute
.
parameter
=
attribute_param
;
result
.
key_attributes
.
push_back
(
key_attribute
);
}
if
(
attributes
->
item
(
i
)
->
parentNode
()
->
nodeName
()
==
"aggregate"
)
{
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
aggregate_nodes
=
attributes
->
item
(
i
)
->
parentNode
()
->
childNodes
();
unsigned
j
;
for
(
j
=
0
;
j
<
aggregate_nodes
->
length
();
j
++
)
{
if
(
aggregate_nodes
->
item
(
j
)
->
nodeName
()
==
"function"
)
{
QueryParseResult
::
Aggregate
aggregate
;
aggregate
.
attribute
=
attribute_name
;
aggregate
.
parameter
=
attribute_param
;
aggregate
.
function
=
aggregate_nodes
->
item
(
j
)
->
innerText
();
result
.
aggregates
.
push_back
(
aggregate
);
break
;
}
}
if
(
j
==
aggregate_nodes
->
length
())
throw
Exception
(
std
::
string
(
"Not found 'function' element for aggregate with attribute "
)
+
attribute_name
,
ErrorCodes
::
NOT_FOUND_FUNCTION_ELEMENT_FOR_AGGREGATE
);
}
if
(
attributes
->
item
(
i
)
->
parentNode
()
->
nodeName
()
==
"condition"
)
{
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
condition_nodes
=
attributes
->
item
(
i
)
->
parentNode
()
->
childNodes
();
QueryParseResult
::
WhereCondition
condition
;
condition
.
attribute
=
attribute_name
;
condition
.
parameter
=
attribute_param
;
unsigned
j
;
for
(
j
=
0
;
j
<
condition_nodes
->
length
();
j
++
)
{
if
(
condition_nodes
->
item
(
j
)
->
nodeName
()
==
"relation"
)
{
condition
.
relation
=
condition_nodes
->
item
(
j
)
->
innerText
();
break
;
}
}
if
(
j
==
condition_nodes
->
length
())
throw
Exception
(
std
::
string
(
"Not found 'relation' element for condition with attribute "
)
+
attribute_name
,
ErrorCodes
::
NOT_FOUND_RELATION_ELEMENT_FOR_CONDITION
);
for
(
j
=
0
;
j
<
condition_nodes
->
length
();
j
++
)
{
if
(
condition_nodes
->
item
(
j
)
->
nodeName
()
==
"rhs"
)
{
condition
.
rhs
=
condition_nodes
->
item
(
j
)
->
innerText
();
break
;
}
}
if
(
j
==
condition_nodes
->
length
())
throw
Exception
(
std
::
string
(
"Not found 'rhs' element for condition with attribute "
)
+
attribute_name
,
ErrorCodes
::
NOT_FOUND_RHS_ELEMENT_FOR_CONDITION
);
result
.
where_conditions
.
push_back
(
condition
);
}
}
if
(
result
.
key_attributes
.
size
()
==
0
)
throw
Exception
(
"No attributes listed."
,
ErrorCodes
::
NO_ATTRIBUTES_LISTED
);
/// получаем условие сортировки
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
sort_nodes
=
result
.
query
->
getElementsByTagName
(
"sort"
);
if
(
sort_nodes
->
length
()
>=
1
)
{
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
column_nodes
=
sort_nodes
->
item
(
0
)
->
childNodes
();
for
(
unsigned
i
=
0
;
i
<
column_nodes
->
length
();
i
++
)
{
if
(
column_nodes
->
item
(
i
)
->
nodeName
()
!=
"column"
)
continue
;
QueryParseResult
::
SortColumn
column
;
column
.
direction
=
"ascending"
;
Poco
::
AutoPtr
<
Poco
::
XML
::
NodeList
>
index_direction_nodes
=
column_nodes
->
item
(
i
)
->
childNodes
();
for
(
unsigned
j
=
0
;
j
<
index_direction_nodes
->
length
();
j
++
)
{
if
(
index_direction_nodes
->
item
(
j
)
->
nodeName
()
==
"index"
)
{
column
.
index
=
DB
::
parse
<
unsigned
>
(
index_direction_nodes
->
item
(
j
)
->
innerText
());
if
(
column
.
index
<
1
||
column
.
index
>
result
.
key_attributes
.
size
()
+
result
.
aggregates
.
size
())
throw
Exception
(
"Index of column in sort clause is out of range."
,
ErrorCodes
::
INDEX_OF_COLUMN_IN_SORT_CLAUSE_IS_OUT_OF_RANGE
);
}
if
(
index_direction_nodes
->
item
(
j
)
->
nodeName
()
==
"direction"
)
{
column
.
direction
=
index_direction_nodes
->
item
(
j
)
->
innerText
();
if
(
column
.
direction
!=
"ascending"
&&
column
.
direction
!=
"descending"
)
throw
Exception
(
"Unknown direction of sorting."
,
ErrorCodes
::
UNKNOWN_DIRECTION_OF_SORTING
);
}
}
result
.
sort_columns
.
push_back
(
column
);
}
}
return
result
;
}
}
}
dbms/src/Server/OLAPQueryParser.h
已删除
100644 → 0
浏览文件 @
c6da35b7
#pragma once
#include <Poco/DOM/DOMParser.h>
#include <Poco/DOM/DOMWriter.h>
#include <Poco/DOM/Document.h>
#include <Poco/DOM/Element.h>
#include <Poco/DOM/Text.h>
#include <Poco/DOM/NodeList.h>
#include <Poco/SAX/InputSource.h>
#include <Poco/RegularExpression.h>
#include <Poco/AutoPtr.h>
#include <common/logger_useful.h>
#include <common/Common.h>
namespace
DB
{
namespace
OLAP
{
/// формат выдачи результата
enum
Format
{
FORMAT_XML
,
FORMAT_TAB
,
FORMAT_BIN
};
/// что делать, если размер результата больше max_result_size
enum
OverflowMode
{
OVERFLOW_MODE_THROW
,
/// прекратить выполнение запроса, вернуть ошибку (по-умолчанию)
OVERFLOW_MODE_BREAK
,
/// вернуть то, что успело посчитаться до переполнения
OVERFLOW_MODE_ANY
,
/** для тех ключей, которые на момент переполнения, попали в результат,
* посчитать до конца, остальные ключи игнорировать
* (то есть, выбрать "первые попавшиеся" max_result_size записи)
*/
};
/// результат парсинга XML-запроса в формате OLAP-server
struct
QueryParseResult
{
struct
KeyAttribute
{
std
::
string
attribute
;
unsigned
parameter
;
};
struct
Aggregate
{
std
::
string
attribute
;
std
::
string
function
;
unsigned
parameter
;
};
struct
WhereCondition
{
std
::
string
attribute
;
unsigned
parameter
;
std
::
string
relation
;
std
::
string
rhs
;
};
struct
SortColumn
{
unsigned
index
;
std
::
string
direction
;
};
/// 0, если не ограничено
unsigned
max_result_size
;
unsigned
max_execution_time
;
unsigned
max_threads_per_counter
;
unsigned
concurrency
;
unsigned
limit
;
/// максимальное количество записей для вывода (все остальные - игнорируются)
bool
cut_date_last
;
bool
cut_dates_for_goals
;
/// если за какую-то дату цели не существовало - то всего лишь пропускать эту дату
/// Использовать таблицу для одного слоя, даже если указан CounterID = 0.
bool
local
;
/// сэмплинг - по какой доле данных выполнять запрос. принимает значения в диапазоне (0, 1]
/// если равно 1 - то отключен
float
sample
;
Format
format
;
bool
has_overflow_mode
=
false
;
OverflowMode
overflow_mode
;
Poco
::
AutoPtr
<
Poco
::
XML
::
Document
>
query
;
CounterID_t
CounterID
;
time_t
date_first
;
time_t
date_last
;
unsigned
days
;
std
::
vector
<
KeyAttribute
>
key_attributes
;
std
::
vector
<
Aggregate
>
aggregates
;
std
::
vector
<
WhereCondition
>
where_conditions
;
std
::
vector
<
SortColumn
>
sort_columns
;
/// Какую иерархию регионов использовать.
std
::
string
regions_point_of_view
;
};
/// Парсер XML-запросов в формате OLAP-server.
class
QueryParser
{
private:
using
AttributeWithParameter
=
std
::
pair
<
std
::
string
,
unsigned
>
;
AttributeWithParameter
parseAttributeWithParameter
(
const
std
::
string
&
s
);
time_t
getLastDate
();
/// regexp для парсинга выражения типа "GoalReaches(111)"
Poco
::
RegularExpression
parse_attribute_with_parameter_regexp
;
Logger
*
log
;
public:
QueryParser
()
:
parse_attribute_with_parameter_regexp
(
"^
\\
s*(
\\
w+)
\\
s*
\\
((
\\
d+)
\\
)
\\
s*$"
),
log
(
&
Logger
::
get
(
"QueryParser"
))
{
}
QueryParseResult
parse
(
std
::
istream
&
s
);
};
}
}
dbms/src/Server/Server.cpp
浏览文件 @
ea734b6b
...
@@ -48,7 +48,6 @@
...
@@ -48,7 +48,6 @@
#include "HTTPHandler.h"
#include "HTTPHandler.h"
#include "ReplicasStatusHandler.h"
#include "ReplicasStatusHandler.h"
#include "InterserverIOHTTPHandler.h"
#include "InterserverIOHTTPHandler.h"
#include "OLAPHTTPHandler.h"
#include "TCPHandler.h"
#include "TCPHandler.h"
#include "MetricsTransmitter.h"
#include "MetricsTransmitter.h"
#include "UsersConfigReloader.h"
#include "UsersConfigReloader.h"
...
@@ -424,30 +423,10 @@ int Server::main(const std::vector<std::string> & args)
...
@@ -424,30 +423,10 @@ int Server::main(const std::vector<std::string> & args)
http_params
);
http_params
);
}
}
/// OLAP HTTP
std
::
experimental
::
optional
<
Poco
::
Net
::
HTTPServer
>
olap_http_server
;
bool
use_olap_server
=
config
().
has
(
"olap_compatibility.port"
);
if
(
use_olap_server
)
{
olap_parser
=
std
::
make_unique
<
OLAP
::
QueryParser
>
();
olap_converter
=
std
::
make_unique
<
OLAP
::
QueryConverter
>
(
config
());
Poco
::
Net
::
ServerSocket
olap_http_socket
(
Poco
::
Net
::
SocketAddress
(
listen_host
,
config
().
getInt
(
"olap_compatibility.port"
)));
olap_http_socket
.
setReceiveTimeout
(
settings
.
receive_timeout
);
olap_http_socket
.
setSendTimeout
(
settings
.
send_timeout
);
olap_http_server
.
emplace
(
new
HTTPRequestHandlerFactory
<
OLAPHTTPHandler
>
(
*
this
,
"OLAPHTTPHandler-factory"
),
server_pool
,
olap_http_socket
,
http_params
);
}
http_server
.
start
();
http_server
.
start
();
tcp_server
.
start
();
tcp_server
.
start
();
if
(
interserver_io_http_server
)
if
(
interserver_io_http_server
)
interserver_io_http_server
->
start
();
interserver_io_http_server
->
start
();
if
(
olap_http_server
)
olap_http_server
->
start
();
LOG_INFO
(
log
,
"Ready for connections."
);
LOG_INFO
(
log
,
"Ready for connections."
);
...
@@ -471,8 +450,6 @@ int Server::main(const std::vector<std::string> & args)
...
@@ -471,8 +450,6 @@ int Server::main(const std::vector<std::string> & args)
http_server
.
stop
();
http_server
.
stop
();
tcp_server
.
stop
();
tcp_server
.
stop
();
if
(
use_olap_server
)
olap_http_server
->
stop
();
);
);
/// try to load dictionaries immediately, throw on error and die
/// try to load dictionaries immediately, throw on error and die
...
...
dbms/src/Server/Server.h
浏览文件 @
ea734b6b
...
@@ -22,17 +22,15 @@
...
@@ -22,17 +22,15 @@
#include <DB/Common/HTMLForm.h>
#include <DB/Common/HTMLForm.h>
#include <DB/Interpreters/Context.h>
#include <DB/Interpreters/Context.h>
#include "OLAPQueryParser.h"
#include "OLAPQueryConverter.h"
/**
Сервер предоставляет три интерфейса
:
/**
Server provides three interfaces
:
* 1. HTTP -
простой интерфейс для доступа из любых приложений
.
* 1. HTTP -
simple interface for any applications
.
* 2. TCP -
интерфейс для доступа из родной библиотеки, родного клиента, и для межсерверного взаимодействия
.
* 2. TCP -
interface for native clickhouse-client and for server to server internal communications
.
*
Более эффективен, так как
*
More rich and efficient, but less compatible
* -
данные передаются по столбцам
;
* -
data is transferred by columns
;
* -
данные передаются со сжатием
;
* -
data is transferred compressed
;
*
Позволяет тонко управлять настройками и получать более подробную информацию в ответах
.
*
Allows to get more information in response
.
* 3.
OLAP-server HTTP - интерфейс для совместимости с устаревшим демоном OLAP-server
.
* 3.
Interserver HTTP - for replication
.
*/
*/
...
@@ -42,12 +40,9 @@ namespace DB
...
@@ -42,12 +40,9 @@ namespace DB
class
Server
:
public
BaseDaemon
class
Server
:
public
BaseDaemon
{
{
public:
public:
///
Глобальные настройки севрера
///
Global settings of server.
std
::
unique_ptr
<
Context
>
global_context
;
std
::
unique_ptr
<
Context
>
global_context
;
std
::
unique_ptr
<
OLAP
::
QueryParser
>
olap_parser
;
std
::
unique_ptr
<
OLAP
::
QueryConverter
>
olap_converter
;
protected:
protected:
void
initialize
(
Application
&
self
)
void
initialize
(
Application
&
self
)
{
{
...
...
dbms/src/Server/TCPHandler.cpp
浏览文件 @
ea734b6b
...
@@ -448,10 +448,7 @@ void TCPHandler::receiveHello()
...
@@ -448,10 +448,7 @@ void TCPHandler::receiveHello()
{
{
writeString
(
"HTTP/1.0 400 Bad Request
\r\n\r\n
"
writeString
(
"HTTP/1.0 400 Bad Request
\r\n\r\n
"
"Port "
+
server
.
config
().
getString
(
"tcp_port"
)
+
" is for clickhouse-client program.
\r\n
"
"Port "
+
server
.
config
().
getString
(
"tcp_port"
)
+
" is for clickhouse-client program.
\r\n
"
"You must use port "
+
server
.
config
().
getString
(
"http_port"
)
+
" for HTTP"
"You must use port "
+
server
.
config
().
getString
(
"http_port"
)
+
" for HTTP.
\r\n
"
,
+
(
server
.
config
().
getBool
(
"use_olap_http_server"
,
false
)
?
"
\r\n
or port "
+
server
.
config
().
getString
(
"olap_http_port"
)
+
" for OLAPServer compatibility layer.
\r\n
"
:
".
\r\n
"
),
*
out
);
*
out
);
throw
Exception
(
"Client has connected to wrong port"
,
ErrorCodes
::
CLIENT_HAS_CONNECTED_TO_WRONG_PORT
);
throw
Exception
(
"Client has connected to wrong port"
,
ErrorCodes
::
CLIENT_HAS_CONNECTED_TO_WRONG_PORT
);
...
...
libs/libcommon/include/common/DateLUTImpl.h
浏览文件 @
ea734b6b
...
@@ -21,8 +21,8 @@
...
@@ -21,8 +21,8 @@
STRONG_TYPEDEF
(
UInt16
,
DayNum_t
);
STRONG_TYPEDEF
(
UInt16
,
DayNum_t
);
/** Lookup
таблица для преобразования времени в дату, а также в месяц или в год или в день недели или в день месяца
.
/** Lookup
table to conversion of time to date, and to month / year / day of week / day of month and so on
.
*
Сейчас она используется для ускорения OLAPServer-а, который делает такие преобразования миллиардами
.
*
First time was implemented for OLAPServer, that needed to do billions of such transformations
.
*/
*/
class
DateLUTImpl
class
DateLUTImpl
{
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录