Skip to content

C
ClickHouse

2dot5 / ClickHouse

通知 3
Star 0
Fork 0
C
ClickHouse

体验新版 GitCode，发现更多精彩内容 >>

提交 310d5225 编写于 作者: A Alexander Tokmakov
浏览文件
操作

minimal implementation

上级 ac54d470
隐藏空白更改
内联 并排
Showing with 276 addition and 104 deletion
cassandra @ 9606ff1f
比较 bc593f26...9606ff1f
Subproject commit bc593f2644a6c50c4057459e242e214a6af70969
Subproject commit 9606ff1f70bd3fc5d395df32e626923c012ffb5f
src/Dictionaries/CassandraBlockInputStream.cpp
浏览文件 @ 310d5225
......@@ -50,76 +50,77 @@ namespace
void insertValue(IColumn & column, const ValueType type, const CassValue * cass_value)
{
/// Cassandra does not support unsigned integers
switch (type)
{
case ValueType::vtUInt8:
{
cass_uint32_t value;
cass_value_get_uint32(cass_value, &value);
static_cast<ColumnUInt8 &>(column).insertValue(value);
cass_int8_t value;
cass_value_get_int8(cass_value, &value);
assert_cast<ColumnUInt8 &>(column).insertValue(value);
break;
}
case ValueType::vtUInt16:
{
cass_uint32_t value;
cass_value_get_uint32(cass_value, &value);
static_cast<ColumnUInt16 &>(column).insertValue(value);
cass_int16_t value;
cass_value_get_int16(cass_value, &value);
assert_cast<ColumnUInt16 &>(column).insertValue(value);
break;
}
case ValueType::vtUInt32:
{
cass_uint32_t value;
cass_value_get_uint32(cass_value, &value);
static_cast<ColumnUInt32 &>(column).insertValue(value);
cass_int32_t value;
cass_value_get_int32(cass_value, &value);
assert_cast<ColumnUInt32 &>(column).insertValue(value);
break;
}
case ValueType::vtUInt64:
{
cass_int64_t value;
cass_value_get_int64(cass_value, &value);
static_cast<ColumnUInt64 &>(column).insertValue(value);
assert_cast<ColumnUInt64 &>(column).insertValue(value);
break;
}
case ValueType::vtInt8:
{
cass_int8_t value;
cass_value_get_int8(cass_value, &value);
static_cast<ColumnInt8 &>(column).insertValue(value);
assert_cast<ColumnInt8 &>(column).insertValue(value);
break;
}
case ValueType::vtInt16:
{
cass_int16_t value;
cass_value_get_int16(cass_value, &value);
static_cast<ColumnInt16 &>(column).insertValue(value);
assert_cast<ColumnInt16 &>(column).insertValue(value);
break;
}
case ValueType::vtInt32:
{
cass_int32_t value;
cass_value_get_int32(cass_value, &value);
static_cast<ColumnInt32 &>(column).insertValue(value);
assert_cast<ColumnInt32 &>(column).insertValue(value);
break;
}
case ValueType::vtInt64:
{
cass_int64_t value;
cass_value_get_int64(cass_value, &value);
static_cast<ColumnInt64 &>(column).insertValue(value);
assert_cast<ColumnInt64 &>(column).insertValue(value);
break;
}
case ValueType::vtFloat32:
{
cass_float_t value;
cass_value_get_float(cass_value, &value);
static_cast<ColumnFloat32 &>(column).insertValue(value);
assert_cast<ColumnFloat32 &>(column).insertValue(value);
break;
}
case ValueType::vtFloat64:
{
cass_double_t value;
cass_value_get_double(cass_value, &value);
static_cast<ColumnFloat64 &>(column).insertValue(value);
assert_cast<ColumnFloat64 &>(column).insertValue(value);
break;
}
case ValueType::vtString:
......@@ -127,21 +128,21 @@ namespace
const char * value;
size_t value_length;
cass_value_get_string(cass_value, &value, &value_length);
static_cast<ColumnString &>(column).insertData(value, value_length);
assert_cast<ColumnString &>(column).insertData(value, value_length);
break;
}
case ValueType::vtDate:
{
cass_int64_t value;
cass_value_get_int64(cass_value, &value);
static_cast<ColumnUInt16 &>(column).insertValue(UInt32{cass_date_from_epoch(value)}); // FIXME
cass_uint32_t value;
cass_value_get_uint32(cass_value, &value);
assert_cast<ColumnUInt16 &>(column).insertValue(static_cast<UInt16>(value));
break;
}
case ValueType::vtDateTime:
{
cass_int64_t value;
cass_value_get_int64(cass_value, &value);
static_cast<ColumnUInt32 &>(column).insertValue(value);
assert_cast<ColumnUInt32 &>(column).insertValue(static_cast<UInt32>(value / 1000));
break;
}
case ValueType::vtUUID:
......@@ -150,7 +151,7 @@ namespace
cass_value_get_uuid(cass_value, &value);
std::array<char, CASS_UUID_STRING_LENGTH> uuid_str;
cass_uuid_string(value, uuid_str.data());
static_cast<ColumnUInt128 &>(column).insert(parse<UUID>(uuid_str.data(), uuid_str.size()));
assert_cast<ColumnUInt128 &>(column).insert(parse<UUID>(uuid_str.data(), uuid_str.size()));
break;
}
}
......@@ -161,10 +162,10 @@ namespace
Block CassandraBlockInputStream::readImpl()
{
if (has_more_pages)
if (!has_more_pages)
return {};
MutableColumns columns(description.sample_block.columns());
MutableColumns columns = description.sample_block.cloneEmptyColumns();
CassFuture* query_future = cass_session_execute(session, statement);
result = cass_future_get_result(query_future);
......@@ -177,23 +178,49 @@ namespace
throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR};
}
const CassRow* row = cass_result_first_row(result);
const CassValue* map = cass_row_get_column(row, 0);
iterator = cass_iterator_from_map(map);
while (cass_iterator_next(iterator)) {
const CassValue* cass_key = cass_iterator_get_map_key(iterator);
const CassValue* cass_value = cass_iterator_get_map_value(iterator);
auto pair_values = {std::make_pair(cass_key, 0ul), std::make_pair(cass_value, 1ul)};
for (const auto &[value, idx]: pair_values) {
if (description.types[idx].second) {
ColumnNullable & column_nullable = static_cast<ColumnNullable &>(*columns[idx]);
insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value);
[[maybe_unused]] size_t row_count = 0;
assert(cass_result_column_count(result) == columns.size());
CassIterator * rows_iter = cass_iterator_from_result(result); /// Points to rows[-1]
while (cass_iterator_next(rows_iter))
{
const CassRow * row = cass_iterator_get_row(rows_iter);
for (size_t col_idx = 0; col_idx < columns.size(); ++col_idx)
{
const CassValue * val = cass_row_get_column(row, col_idx);
if (cass_value_is_null(val))
columns[col_idx]->insertDefault();
else if (description.types[col_idx].second)
{
ColumnNullable & column_nullable = static_cast<ColumnNullable &>(*columns[col_idx]);
insertValue(column_nullable.getNestedColumn(), description.types[col_idx].first, val);
column_nullable.getNullMapData().emplace_back(0);
} else {
insertValue(*columns[idx], description.types[idx].first, value);
}
else
insertValue(*columns[col_idx], description.types[col_idx].first, val);
}
++row_count;
}
assert(cass_result_row_count(result) == row_count);
cass_iterator_free(rows_iter);
//const CassRow* row = cass_result_first_row(result);
//const CassValue* map = cass_row_get_column(row, 0);
//const CassValue* map = cass_row_get_column(row, 0);
//iterator = cass_iterator_from_map(map);
//while (cass_iterator_next(iterator)) {
// const CassValue* cass_key = cass_iterator_get_map_key(iterator);
// const CassValue* cass_value = cass_iterator_get_map_value(iterator);
// auto pair_values = {std::make_pair(cass_key, 0ul), std::make_pair(cass_value, 1ul)};
// for (const auto &[value, idx]: pair_values) {
// if (description.types[idx].second) {
// ColumnNullable & column_nullable = static_cast<ColumnNullable &>(*columns[idx]);
// insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value);
// column_nullable.getNullMapData().emplace_back(0);
// } else {
// insertValue(*columns[idx], description.types[idx].first, value);
// }
// }
//}
has_more_pages = cass_result_has_more_pages(result);
......@@ -207,5 +234,28 @@ namespace
}
void cassandraCheck(CassError code)
{
if (code != CASS_OK)
throw Exception("Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code),
ErrorCodes::CASSANDRA_INTERNAL_ERROR);
}
void cassandraWaitAndCheck(CassFuture * future)
{
auto code = cass_future_error_code(future); /// Waits if not ready
if (code == CASS_OK)
{
cass_future_free(future);
return;
}
const char * message;
size_t message_len;
cass_future_error_message(future, &message, & message_len);
String full_message = "Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code) + ": " + message;
cass_future_free(future); /// Frees message
throw Exception(full_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR);
}
}
#endif
src/Dictionaries/CassandraBlockInputStream.h
浏览文件 @ 310d5225
......@@ -8,6 +8,11 @@
namespace DB
{
void cassandraCheck(CassError error);
void cassandraWaitAndCheck(CassFuture * future);
/// Allows processing results of a Cassandra query as a sequence of Blocks, simplifies chaining
class CassandraBlockInputStream final : public IBlockInputStream
{
......
src/Dictionaries/CassandraDictionarySource.cpp
浏览文件 @ 310d5225
#include "CassandraDictionarySource.h"
#include "DictionarySourceFactory.h"
#include "DictionaryStructure.h"
#include "ExternalQueryBuilder.h"
#include <common/logger_useful.h>
namespace DB
{
......@@ -51,25 +53,32 @@ static const size_t max_block_size = 8192;
CassandraDictionarySource::CassandraDictionarySource(
const DB::DictionaryStructure & dict_struct_,
const std::string & host_,
const String & host_,
UInt16 port_,
const std::string & user_,
const std::string & password_,
const std::string & method_,
const std::string & db_,
const String & user_,
const String & password_,
//const std::string & method_,
const String & db_,
const String & table_,
const DB::Block & sample_block_)
: dict_struct(dict_struct_)
: log(&Logger::get("CassandraDictionarySource"))
, dict_struct(dict_struct_)
, host(host_)
, port(port_)
, user(user_)
, password(password_)
, method(method_)
//, method(method_)
, db(db_)
, table(table_)
, sample_block(sample_block_)
, cluster(cass_cluster_new())
, cluster(cass_cluster_new()) //FIXME will not be freed in case of exception
, session(cass_session_new())
{
cass_cluster_set_contact_points(cluster, toConnectionString(host, port).c_str());
cassandraCheck(cass_cluster_set_contact_points(cluster, host.c_str()));
if (port)
cassandraCheck(cass_cluster_set_port(cluster, port));
cass_cluster_set_credentials(cluster, user.c_str(), password.c_str());
cassandraWaitAndCheck(cass_session_connect_keyspace(session, cluster, db.c_str()));
}
CassandraDictionarySource::CassandraDictionarySource(
......@@ -80,11 +89,12 @@ CassandraDictionarySource::CassandraDictionarySource(
: CassandraDictionarySource(
dict_struct_,
config.getString(config_prefix + ".host"),
config.getUInt(config_prefix + ".port"),
config.getUInt(config_prefix + ".port", 0),
config.getString(config_prefix + ".user", ""),
config.getString(config_prefix + ".password", ""),
config.getString(config_prefix + ".method", ""),
config.getString(config_prefix + ".db", ""),
//config.getString(config_prefix + ".method", ""),
config.getString(config_prefix + ".keyspace", ""),
config.getString(config_prefix + ".column_family"),
sample_block_)
{
}
......@@ -95,8 +105,9 @@ CassandraDictionarySource::CassandraDictionarySource(const CassandraDictionarySo
other.port,
other.user,
other.password,
other.method,
//other.method,
other.db,
other.table,
other.sample_block}
{
}
......@@ -106,18 +117,45 @@ CassandraDictionarySource::~CassandraDictionarySource() {
cass_cluster_free(cluster);
}
std::string CassandraDictionarySource::toConnectionString(const std::string &host, const UInt16 port) {
return host + (port != 0 ? ":" + std::to_string(port) : "");
}
//std::string CassandraDictionarySource::toConnectionString(const std::string &host, const UInt16 port) {
// return host + (port != 0 ? ":" + std::to_string(port) : "");
//}
BlockInputStreamPtr CassandraDictionarySource::loadAll() {
return std::make_shared<CassandraBlockInputStream>(nullptr, "", sample_block, max_block_size);
BlockInputStreamPtr CassandraDictionarySource::loadAll()
{
ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes};
String query = builder.composeLoadAllQuery();
query.pop_back();
query += " ALLOW FILTERING;";
LOG_INFO(log, "Loading all using query: " << query);
return std::make_shared<CassandraBlockInputStream>(session, query, sample_block, max_block_size);
}
std::string CassandraDictionarySource::toString() const {
return "Cassandra: " + /*db + '.' + collection + ',' + (user.empty() ? " " : " " + user + '@') + */ host + ':' + DB::toString(port);
}
BlockInputStreamPtr CassandraDictionarySource::loadIds(const std::vector<UInt64> & ids)
{
ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes};
String query = builder.composeLoadIdsQuery(ids);
query.pop_back();
query += " ALLOW FILTERING;";
LOG_INFO(log, "Loading ids using query: " << query);
return std::make_shared<CassandraBlockInputStream>(session, query, sample_block, max_block_size);
}
BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_columns, const std::vector<size_t> & requested_rows)
{
//FIXME split conditions on partition key and clustering key
ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes};
String query = builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::IN_WITH_TUPLES);
query.pop_back();
query += " ALLOW FILTERING;";
LOG_INFO(log, "Loading keys using query: " << query);
return std::make_shared<CassandraBlockInputStream>(session, query, sample_block, max_block_size);
}
}
......
src/Dictionaries/CassandraDictionarySource.h
浏览文件 @ 310d5225
#pragma once
#if !defined(ARCADIA_BUILD)
#include <Common/config.h>
#include <Core/Block.h>
#endif
#if USE_CASSANDRA
# include "DictionaryStructure.h"
# include "IDictionarySource.h"
# include <cassandra.h>
#include "DictionaryStructure.h"
#include "IDictionarySource.h"
#include <Core/Block.h>
#include <Poco/Logger.h>
#include <cassandra.h>
namespace DB
{
class CassandraDictionarySource final : public IDictionarySource {
CassandraDictionarySource(
const DictionaryStructure & dict_struct,
const std::string & host,
const String & host,
UInt16 port,
const std::string & user,
const std::string & password,
const std::string & method,
const std::string & db,
const String & user,
const String & password,
//const std::string & method,
const String & db,
const String & table,
const Block & sample_block);
public:
......@@ -44,15 +48,15 @@ public:
DictionarySourcePtr clone() const override { return std::make_unique<CassandraDictionarySource>(*this); }
BlockInputStreamPtr loadIds(const std::vector<UInt64> & /* ids */) override
{
throw Exception{"Method loadIds is not implemented yet", ErrorCodes::NOT_IMPLEMENTED};
}
BlockInputStreamPtr loadIds(const std::vector<UInt64> & ids) override;
//{
// throw Exception{"Method loadIds is not implemented yet", ErrorCodes::NOT_IMPLEMENTED};
//}
BlockInputStreamPtr loadKeys(const Columns & /* key_columns */, const std::vector<size_t> & /* requested_rows */) override
{
throw Exception{"Method loadKeys is not implemented yet", ErrorCodes::NOT_IMPLEMENTED};
}
BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector<size_t> & requested_rows) override;
//{
// throw Exception{"Method loadKeys is not implemented yet", ErrorCodes::NOT_IMPLEMENTED};
//}
BlockInputStreamPtr loadUpdatedAll() override
{
......@@ -62,15 +66,17 @@ public:
std::string toString() const override;
private:
static std::string toConnectionString(const std::string & host, const UInt16 port);
//static std::string toConnectionString(const std::string & host, const UInt16 port);
Poco::Logger * log;
const DictionaryStructure dict_struct;
const std::string host;
const String host;
const UInt16 port;
const std::string user;
const std::string password;
const std::string method;
const std::string db;
const String user;
const String password;
//const std::string method;
const String db;
const String table;
Block sample_block;
CassCluster * cluster;
......
src/Dictionaries/ExternalQueryBuilder.cpp
浏览文件 @ 310d5225
......@@ -63,6 +63,13 @@ void ExternalQueryBuilder::writeQuoted(const std::string & s, WriteBuffer & out)
std::string ExternalQueryBuilder::composeLoadAllQuery() const
{
WriteBufferFromOwnString out;
composeLoadAllQuery(out);
writeChar(';', out);
return out.str();
}
void ExternalQueryBuilder::composeLoadAllQuery(WriteBuffer & out) const
{
writeString("SELECT ", out);
if (dict_struct.id)
......@@ -149,24 +156,26 @@ std::string ExternalQueryBuilder::composeLoadAllQuery() const
writeString(" WHERE ", out);
writeString(where, out);
}
writeChar(';', out);
return out.str();
}
std::string ExternalQueryBuilder::composeUpdateQuery(const std::string & update_field, const std::string & time_point) const
{
std::string out = composeLoadAllQuery();
std::string update_query;
WriteBufferFromOwnString out;
composeLoadAllQuery(out);
if (!where.empty())
update_query = " AND " + update_field + " >= '" + time_point + "'";
writeString(" AND ", out);
else
update_query = " WHERE " + update_field + " >= '" + time_point + "'";
writeString(" WHERE ", out);
writeQuoted(update_field, out);
writeString(" >= '", out);
writeString(time_point, out);
writeChar('\'', out);
return out.insert(out.size() - 1, update_query); /// This is done to insert "update_query" before "out"'s semicolon
writeChar(';', out);
return out.str();
}
......@@ -303,7 +312,7 @@ ExternalQueryBuilder::composeLoadKeysQuery(const Columns & key_columns, const st
}
else /* if (method == IN_WITH_TUPLES) */
{
writeString(composeKeyTupleDefinition(), out);
composeKeyTupleDefinition(out);
writeString(" IN (", out);
first = true;
......@@ -346,7 +355,7 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons
const auto & key_description = (*dict_struct.key)[i];
/// key_i=value_i
writeString(key_description.name, out);
writeQuoted(key_description.name, out);
writeString("=", out);
key_description.type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings);
}
......@@ -355,26 +364,24 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons
}
std::string ExternalQueryBuilder::composeKeyTupleDefinition() const
void ExternalQueryBuilder::composeKeyTupleDefinition(WriteBuffer & out) const
{
if (!dict_struct.key)
throw Exception{"Composite key required for method", ErrorCodes::UNSUPPORTED_METHOD};
std::string result{"("};
writeChar('(', out);
auto first = true;
for (const auto & key : *dict_struct.key)
{
if (!first)
result += ", ";
writeString(", ", out);
first = false;
result += key.name;
writeQuoted(key.name, out);
}
result += ")";
return result;
writeChar(')', out);
}
......
src/Dictionaries/ExternalQueryBuilder.h
浏览文件 @ 310d5225
......@@ -58,11 +58,13 @@ struct ExternalQueryBuilder
private:
const FormatSettings format_settings;
void composeLoadAllQuery(WriteBuffer & out) const;
/// Expression in form (x = c1 AND y = c2 ...)
void composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out) const;
/// Expression in form (x, y, ...)
std::string composeKeyTupleDefinition() const;
void composeKeyTupleDefinition(WriteBuffer & out) const;
/// Expression in form (c1, c2, ...)
void composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out) const;
......
tests/integration/helpers/cluster.py
浏览文件 @ 310d5225
......@@ -19,6 +19,7 @@ import pprint
import psycopg2
import pymongo
import pymysql
import cassandra.cluster
from dicttoxml import dicttoxml
from kazoo.client import KazooClient
from kazoo.exceptions import KazooException
......@@ -448,6 +449,18 @@ class ClickHouseCluster:
logging.warning("Can't connect to SchemaRegistry: %s", str(ex))
time.sleep(1)
def wait_cassandra_to_start(self, timeout=15):
cass_client = cassandra.cluster.Cluster(["localhost"], port="9043")
start = time.time()
while time.time() - start < timeout:
try:
cass_client.connect().execute("drop keyspace if exists test;")
logging.info("Connected to Cassandra %s")
return
except Exception as ex:
logging.warning("Can't connect to Minio: %s", str(ex))
time.sleep(1)
def start(self, destroy_dirs=True):
if self.is_up:
return
......@@ -526,7 +539,7 @@ class ClickHouseCluster:
if self.with_cassandra and self.base_cassandra_cmd:
subprocess_check_call(self.base_cassandra_cmd + ['up', '-d', '--force-recreate'])
time.sleep(10)
self.wait_cassandra_to_start()
clickhouse_start_cmd = self.base_cmd + ['up', '-d', '--no-recreate']
logging.info("Trying to create ClickHouse instance by command %s", ' '.join(map(str, clickhouse_start_cmd)))
......
tests/integration/helpers/docker_compose_cassandra.yml
浏览文件 @ 310d5225
......@@ -4,4 +4,4 @@ services:
image: cassandra
restart: always
ports:
- 6340:6349
- 9043:9042
tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py
浏览文件 @ 310d5225
......@@ -8,6 +8,7 @@ import aerospike
from tzlocal import get_localzone
import datetime
import os
import uuid
class ExternalSource(object):
......@@ -407,23 +408,70 @@ class SourceHTTPS(SourceHTTPBase):
return "https"
class SourceCassandra(ExternalSource):
TYPE_MAPPING = {
'UInt8': 'tinyint',
'UInt16': 'smallint',
'UInt32': 'int',
'UInt64': 'bigint',
'Int8': 'tinyint',
'Int16': 'smallint',
'Int32': 'int',
'Int64': 'bigint',
'UUID': 'uuid',
'Date': 'date',
'DateTime': 'timestamp',
'String': 'text',
'Float32': 'float',
'Float64': 'double'
}
def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password):
ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password)
self.structure = dict()
def get_source_str(self, table_name):
return '''
<cassandra>
<host>{host}</host>
<port>{port}</port>
<keyspace>test</keyspace>
<column_family>{table}</column_family>
</cassandra>
'''.format(
host=self.docker_hostname,
port=self.docker_port,
table=table_name,
)
def prepare(self, structure, table_name, cluster):
self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port)
self.session = self.client.connect()
self.session.execute("create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};")
self.structure[table_name] = structure
columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()]
keys = ['"' + col.name + '"' for col in structure.keys]
# FIXME use partition key
query = 'create table test."{name}" ({columns}, primary key ("{some_col}", {pk}));'.format(
name=table_name, columns=', '.join(columns), some_col=structure.ordinary_fields[0].name, pk=', '.join(keys))
self.session.execute(query)
self.prepared = True
def get_value_to_insert(self, value, type):
if type == 'UUID':
return uuid.UUID(value)
elif type == 'DateTime':
local_datetime = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
return get_localzone().localize(local_datetime)
return value
def load_data(self, data, table_name):
pass
names_and_types = [(field.name, field.field_type) for field in self.structure[table_name].get_all_fields()]
columns = ['"' + col[0] + '"' for col in names_and_types]
insert = 'insert into test."{table}" ({columns}) values ({args})'.format(
table=table_name, columns=','.join(columns), args=','.join(['%s']*len(columns)))
for row in data:
values = [self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) for col in names_and_types]
self.session.execute(insert, values)
class SourceRedis(ExternalSource):
def __init__(
......
tests/integration/test_dictionaries_all_layouts_and_sources/test.py
浏览文件 @ 310d5225
......@@ -8,7 +8,7 @@ from external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceEx
from external_sources import SourceMongo, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra
from external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra
import math
import time
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries')
......@@ -119,7 +119,7 @@ LAYOUTS = [
]
SOURCES = [
SourceCassandra("Cassandra", "localhost", "6340", "cassandra1", "6349", "", ""),
SourceCassandra("Cassandra", "localhost", "9043", "cassandra1", "9042", "", ""),
SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"),
SourceMongoURI("MongoDB_URI", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"),
SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse"),
......@@ -134,7 +134,7 @@ SOURCES = [
DICTIONARIES = []
# Key-value dictionaries with onle one possible field for key
# Key-value dictionaries with only one possible field for key
SOURCES_KV = [
SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "", storage_type="simple"),
SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "", storage_type="hash_map"),
......@@ -212,6 +212,7 @@ def get_dictionaries(fold, total_folds, all_dicts):
return all_dicts[fold * chunk_len : (fold + 1) * chunk_len]
#@pytest.mark.timeout(3000)
@pytest.mark.parametrize("fold", list(range(10)))
def test_simple_dictionaries(started_cluster, fold):
fields = FIELDS["simple"]
......@@ -227,6 +228,8 @@ def test_simple_dictionaries(started_cluster, fold):
node.query("system reload dictionaries")
#time.sleep(3000)
queries_with_answers = []
for dct in simple_dicts:
for row in data:
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或