diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 3f93313b73070e30336e06b322d187d87995e23a..632c754a8d193e0e38b105640cec004720a3b602 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -367,6 +367,8 @@ namespace ErrorCodes extern const int CANNOT_ASSIGN_OPTIMIZE = 388; extern const int INSERT_WAS_DEDUPLICATED = 389; extern const int CANNOT_GET_CREATE_TABLE_QUERY = 390; + extern const int EXTERNAL_LIBRARY_ERROR = 391; + extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Dictionaries/LibraryDictionarySource.cpp b/dbms/src/Dictionaries/LibraryDictionarySource.cpp index 9015712a322e59617a2c3f0ced5e3b3114454cdb..457996696ccfa9e55fe049d877f5b2dead015592 100644 --- a/dbms/src/Dictionaries/LibraryDictionarySource.cpp +++ b/dbms/src/Dictionaries/LibraryDictionarySource.cpp @@ -14,6 +14,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; extern const int FILE_DOESNT_EXIST; + extern const int EXTERNAL_LIBRARY_ERROR; } @@ -71,7 +72,11 @@ namespace if (!data) return sample_block.cloneEmpty(); - auto columns_received = static_cast(data); + auto columns_received = static_cast(data); + if (columns_received->error_code) + throw Exception("Received error: " + std::to_string(columns_received->error_code) + " " + + (columns_received->error_string ? columns_received->error_string : ""), + ErrorCodes::EXTERNAL_LIBRARY_ERROR); MutableColumns columns(sample_block.columns()); for (const auto i : ext::range(0, columns.size())) @@ -86,17 +91,12 @@ namespace for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n) { - const auto & type = sample_block.getByPosition(row_n).type; - if (type->isStringOrFixedString()) - { - const auto & data = ext::bit_cast(columns_received->data[col_n].data[row_n]); - auto len = strlen(data); - columns[row_n]->insertData(data, len); - } - else - { - columns[row_n]->insert(columns_received->data[col_n].data[row_n]); - } + const auto & field = columns_received->data[col_n].data[row_n]; + if (!field.data) + continue; + const auto & size = field.size; + const auto & data = static_cast(field.data); + columns[row_n]->insertData(data, size); } } @@ -157,11 +157,11 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() /// Get function pointer before dataAllocate call because library->get may throw. auto fptr - = library->getstrings), decltype(&columns))>("ClickHouseDictionary_v1_loadAll"); - data_ptr = library->get("ClickHouseDictionary_v1_dataAllocate")(); + = library->getstrings), decltype(&columns))>("ClickHouseDictionary_v2_loadAll"); + data_ptr = library->get("ClickHouseDictionary_v2_dataAllocate")(); auto data = fptr(data_ptr, &settings->strings, &columns); auto block = dataToBlock(description.sample_block, data); - library->get("ClickHouseDictionary_v1_dataDelete")(data_ptr); + library->get("ClickHouseDictionary_v2_dataDelete")(data_ptr); return std::make_shared(block); } @@ -183,11 +183,11 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & /// Get function pointer before dataAllocate call because library->get may throw. auto fptr = library->getstrings), decltype(&columns_pass), decltype(&ids_data))>( - "ClickHouseDictionary_v1_loadIds"); - data_ptr = library->get("ClickHouseDictionary_v1_dataAllocate")(); + "ClickHouseDictionary_v2_loadIds"); + data_ptr = library->get("ClickHouseDictionary_v2_dataAllocate")(); auto data = fptr(data_ptr, &settings->strings, &columns_pass, &ids_data); auto block = dataToBlock(description.sample_block, data); - library->get("ClickHouseDictionary_v1_dataDelete")(data_ptr); + library->get("ClickHouseDictionary_v2_dataDelete")(data_ptr); return std::make_shared(block); } @@ -195,16 +195,6 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column { LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size()); - /* - auto columns_c = std::make_unique(key_columns.size() + 1); - size_t i = 0; - for (auto & column : key_columns) - { - columns_c[i] = column->getName().c_str(); - ++i; - } - columns_c[i] = nullptr; - */ auto columns_holder = std::make_unique(key_columns.size()); ClickHouseLibrary::CStrings columns_pass{ static_cast(columns_holder.get()), key_columns.size()}; @@ -221,17 +211,17 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column /// Get function pointer before dataAllocate call because library->get may throw. auto fptr = library->getstrings), decltype(&columns_pass), decltype(&requested_rows_c))>( - "ClickHouseDictionary_v1_loadKeys"); - data_ptr = library->get("ClickHouseDictionary_v1_dataAllocate")(); + "ClickHouseDictionary_v2_loadKeys"); + data_ptr = library->get("ClickHouseDictionary_v2_dataAllocate")(); auto data = fptr(data_ptr, &settings->strings, &columns_pass, &requested_rows_c); auto block = dataToBlock(description.sample_block, data); - library->get("ClickHouseDictionary_v1_dataDelete")(data_ptr); + library->get("ClickHouseDictionary_v2_dataDelete")(data_ptr); return std::make_shared(block); } bool LibraryDictionarySource::isModified() const { - auto fptr = library->tryGetstrings))>("ClickHouseDictionary_v1_isModified"); + auto fptr = library->tryGetstrings))>("ClickHouseDictionary_v2_isModified"); if (fptr) return fptr(&settings->strings); return true; @@ -239,7 +229,7 @@ bool LibraryDictionarySource::isModified() const bool LibraryDictionarySource::supportsSelectiveLoad() const { - auto fptr = library->tryGetstrings))>("ClickHouseDictionary_v1_supportsSelectiveLoad"); + auto fptr = library->tryGetstrings))>("ClickHouseDictionary_v2_supportsSelectiveLoad"); if (fptr) return fptr(&settings->strings); return true; diff --git a/dbms/src/Dictionaries/LibraryDictionarySourceExternal.h b/dbms/src/Dictionaries/LibraryDictionarySourceExternal.h index 3a7af5acfce6af80e58b0a66a3f455df06a66fd6..f510c927d40b09ebe6cd7e851e9e9f8e0b4aa5f5 100644 --- a/dbms/src/Dictionaries/LibraryDictionarySourceExternal.h +++ b/dbms/src/Dictionaries/LibraryDictionarySourceExternal.h @@ -2,6 +2,8 @@ #include +#define CLICKHOUSE_DICTIONARY_LIBRARY_API 1 + namespace ClickHouseLibrary { using CString = const char *; @@ -25,4 +27,24 @@ struct ColumnsUInt64 VectorUInt64 * data = nullptr; uint64_t size = 0; }; + +struct Field +{ + const void * data = nullptr; + uint64_t size = 0; +}; + +struct Row +{ + const Field * data = nullptr; + uint64_t size = 0; +}; + +struct Table +{ + const Row * data = nullptr; + uint64_t size = 0; + uint64_t error_code = 0; // 0 = ok; !0 = error, with message in error_string + const char * error_string = nullptr; +}; } diff --git a/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp b/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp index 97fb81585673706845c33066399c0ce09ca4ec5e..3c759ba336e5e839412c5378e432da628e29f27c 100644 --- a/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp +++ b/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp @@ -17,11 +17,12 @@ struct DataHolder ClickHouseLibrary::ColumnsUInt64 columns; }; -extern "C" -{ +extern "C" { -void * ClickHouseDictionary_v1_loadIds( - void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * columns, const struct ClickHouseLibrary::VectorUInt64 * ids) +void * ClickHouseDictionary_v2_loadIds(void * data_ptr, + ClickHouseLibrary::CStrings * settings, + ClickHouseLibrary::CStrings * columns, + const struct ClickHouseLibrary::VectorUInt64 * ids) { auto ptr = static_cast(data_ptr); @@ -73,7 +74,7 @@ void * ClickHouseDictionary_v1_loadIds( return nullptr; } -void * ClickHouseDictionary_v1_loadAll(void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * /*columns*/) +void * ClickHouseDictionary_v2_loadAll(void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * /*columns*/) { auto ptr = static_cast(data_ptr); std::cerr << "loadAll lib call ptr=" << data_ptr << " => " << ptr << "\n"; @@ -113,7 +114,7 @@ void * ClickHouseDictionary_v1_loadAll(void * data_ptr, ClickHouseLibrary::CStri return nullptr; } -void * ClickHouseDictionary_v1_loadKeys(void * data_ptr, +void * ClickHouseDictionary_v2_loadKeys(void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * columns, const ClickHouseLibrary::VectorUInt64 * requested_rows) @@ -148,13 +149,13 @@ void * ClickHouseDictionary_v1_loadKeys(void * data_ptr, return nullptr; } -void * ClickHouseDictionary_v1_dataAllocate() +void * ClickHouseDictionary_v2_dataAllocate() { auto data_ptr = new DataHolder; return data_ptr; } -void ClickHouseDictionary_v1_dataDelete(void * data_ptr) +void ClickHouseDictionary_v2_dataDelete(void * data_ptr) { auto ptr = static_cast(data_ptr); delete ptr; diff --git a/libs/libcommon/include/common/iostream_debug_helpers.h b/libs/libcommon/include/common/iostream_debug_helpers.h index e590f8e2ff609b17f9fbfd0b339f7aec3558570b..a33cfefe10101b062230ca54418171811f8c7266 100644 --- a/libs/libcommon/include/common/iostream_debug_helpers.h +++ b/libs/libcommon/include/common/iostream_debug_helpers.h @@ -22,6 +22,7 @@ #define DUMP3(V1, V2, V3) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3) << "\n"; #define DUMP4(V1, V2, V3, V4) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3)<< ", " << DUMPS(V4) << "\n"; #define DUMP5(V1, V2, V3, V4, V5) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3)<< ", " << DUMPS(V4) << ", " << DUMPS(V5) << "\n"; +#define DUMP6(V1, V2, V3, V4, V5, V6) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3)<< ", " << DUMPS(V4) << ", " << DUMPS(V5) << ", " << DUMPS(V6) << "\n"; namespace std