LibraryDictionarySource: api v2 with strings (#1915)

* LibraryDictionarySource: Support numeric types, initial support for strings

dbms/src/Common/ErrorCodes.cpp

@@ -367,6 +367,8 @@ namespace ErrorCodes
     extern const int CANNOT_ASSIGN_OPTIMIZE = 388;
     extern const int INSERT_WAS_DEDUPLICATED = 389;
     extern const int CANNOT_GET_CREATE_TABLE_QUERY = 390;
+    extern const int EXTERNAL_LIBRARY_ERROR = 391;
+
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Dictionaries/LibraryDictionarySource.cpp

@@ -14,6 +14,7 @@ namespace ErrorCodes
     extern const int NOT_IMPLEMENTED;
     extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
     extern const int FILE_DOESNT_EXIST;
+    extern const int EXTERNAL_LIBRARY_ERROR;
 }
 
 
@@ -71,7 +72,11 @@ namespace
         if (!data)
             return sample_block.cloneEmpty();
 
-        auto columns_received = static_cast<const ClickHouseLibrary::ColumnsUInt64 *>(data);
+        auto columns_received = static_cast<const ClickHouseLibrary::Table *>(data);
+        if (columns_received->error_code)
+            throw Exception("Received error: " + std::to_string(columns_received->error_code) + " "
+                    + (columns_received->error_string ? columns_received->error_string : ""),
+                ErrorCodes::EXTERNAL_LIBRARY_ERROR);
 
         MutableColumns columns(sample_block.columns());
         for (const auto i : ext::range(0, columns.size()))
@@ -86,17 +91,12 @@ namespace
 
             for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n)
             {
-                const auto & type = sample_block.getByPosition(row_n).type;
-                if (type->isStringOrFixedString())
-                {
-                    const auto & data = ext::bit_cast<const char *>(columns_received->data[col_n].data[row_n]);
-                    auto len = strlen(data);
-                    columns[row_n]->insertData(data, len);
-                }
-                else
-                {
-                    columns[row_n]->insert(columns_received->data[col_n].data[row_n]);
-                }
+                const auto & field = columns_received->data[col_n].data[row_n];
+                if (!field.data)
+                    continue;
+                const auto & size = field.size;
+                const auto & data = static_cast<const char *>(field.data);
+                columns[row_n]->insertData(data, size);
             }
         }
 
@@ -157,11 +157,11 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll()
 
     /// Get function pointer before dataAllocate call because library->get may throw.
     auto fptr
-        = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns))>("ClickHouseDictionary_v1_loadAll");
-    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v1_dataAllocate")();
+        = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns))>("ClickHouseDictionary_v2_loadAll");
+    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v2_dataAllocate")();
     auto data = fptr(data_ptr, &settings->strings, &columns);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void *)>("ClickHouseDictionary_v1_dataDelete")(data_ptr);
+    library->get<void (*)(void *)>("ClickHouseDictionary_v2_dataDelete")(data_ptr);
     return std::make_shared<OneBlockInputStream>(block);
 }
 
@@ -183,11 +183,11 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector<UInt64> &
 
     /// Get function pointer before dataAllocate call because library->get may throw.
     auto fptr = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&ids_data))>(
-        "ClickHouseDictionary_v1_loadIds");
-    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v1_dataAllocate")();
+        "ClickHouseDictionary_v2_loadIds");
+    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v2_dataAllocate")();
     auto data = fptr(data_ptr, &settings->strings, &columns_pass, &ids_data);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void * data_ptr)>("ClickHouseDictionary_v1_dataDelete")(data_ptr);
+    library->get<void (*)(void * data_ptr)>("ClickHouseDictionary_v2_dataDelete")(data_ptr);
     return std::make_shared<OneBlockInputStream>(block);
 }
 
@@ -195,16 +195,6 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column
 {
     LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size());
 
-    /*
-    auto columns_c = std::make_unique<ClickHouseLibrary::Columns>(key_columns.size() + 1);
-    size_t i = 0;
-    for (auto & column : key_columns)
-    {
-        columns_c[i] = column->getName().c_str();
-        ++i;
-    }
-    columns_c[i] = nullptr;
-    */
     auto columns_holder = std::make_unique<ClickHouseLibrary::CString[]>(key_columns.size());
     ClickHouseLibrary::CStrings columns_pass{
         static_cast<decltype(ClickHouseLibrary::CStrings::data)>(columns_holder.get()), key_columns.size()};
@@ -221,17 +211,17 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column
     /// Get function pointer before dataAllocate call because library->get may throw.
     auto fptr
         = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&requested_rows_c))>(
-            "ClickHouseDictionary_v1_loadKeys");
-    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v1_dataAllocate")();
+            "ClickHouseDictionary_v2_loadKeys");
+    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v2_dataAllocate")();
     auto data = fptr(data_ptr, &settings->strings, &columns_pass, &requested_rows_c);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void * data_ptr)>("ClickHouseDictionary_v1_dataDelete")(data_ptr);
+    library->get<void (*)(void * data_ptr)>("ClickHouseDictionary_v2_dataDelete")(data_ptr);
     return std::make_shared<OneBlockInputStream>(block);
 }
 
 bool LibraryDictionarySource::isModified() const
 {
-    auto fptr = library->tryGet<void * (*)(decltype(&settings->strings))>("ClickHouseDictionary_v1_isModified");
+    auto fptr = library->tryGet<void * (*)(decltype(&settings->strings))>("ClickHouseDictionary_v2_isModified");
     if (fptr)
         return fptr(&settings->strings);
     return true;
@@ -239,7 +229,7 @@ bool LibraryDictionarySource::isModified() const
 
 bool LibraryDictionarySource::supportsSelectiveLoad() const
 {
-    auto fptr = library->tryGet<void * (*)(decltype(&settings->strings))>("ClickHouseDictionary_v1_supportsSelectiveLoad");
+    auto fptr = library->tryGet<void * (*)(decltype(&settings->strings))>("ClickHouseDictionary_v2_supportsSelectiveLoad");
     if (fptr)
         return fptr(&settings->strings);
     return true;

dbms/src/Dictionaries/LibraryDictionarySourceExternal.h

@@ -2,6 +2,8 @@
 
 #include <cstdint>
 
+#define CLICKHOUSE_DICTIONARY_LIBRARY_API 1
+
 namespace ClickHouseLibrary
 {
 using CString = const char *;
@@ -25,4 +27,24 @@ struct ColumnsUInt64
     VectorUInt64 * data = nullptr;
     uint64_t size = 0;
 };
+
+struct Field
+{
+    const void * data = nullptr;
+    uint64_t size = 0;
+};
+
+struct Row
+{
+    const Field * data = nullptr;
+    uint64_t size = 0;
+};
+
+struct Table
+{
+    const Row * data = nullptr;
+    uint64_t size = 0;
+    uint64_t error_code = 0; // 0 = ok; !0 = error, with message in error_string
+    const char * error_string = nullptr;
+};
 }

dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp

@@ -17,11 +17,12 @@ struct DataHolder
     ClickHouseLibrary::ColumnsUInt64 columns;
 };
 
-extern "C"
-{
+extern "C" {
 
-void * ClickHouseDictionary_v1_loadIds(
-    void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * columns, const struct ClickHouseLibrary::VectorUInt64 * ids)
+void * ClickHouseDictionary_v2_loadIds(void * data_ptr,
+    ClickHouseLibrary::CStrings * settings,
+    ClickHouseLibrary::CStrings * columns,
+    const struct ClickHouseLibrary::VectorUInt64 * ids)
 {
     auto ptr = static_cast<DataHolder *>(data_ptr);
 
@@ -73,7 +74,7 @@ void * ClickHouseDictionary_v1_loadIds(
     return nullptr;
 }
 
-void * ClickHouseDictionary_v1_loadAll(void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * /*columns*/)
+void * ClickHouseDictionary_v2_loadAll(void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * /*columns*/)
 {
     auto ptr = static_cast<DataHolder *>(data_ptr);
     std::cerr << "loadAll lib call ptr=" << data_ptr << " => " << ptr << "\n";
@@ -113,7 +114,7 @@ void * ClickHouseDictionary_v1_loadAll(void * data_ptr, ClickHouseLibrary::CStri
     return nullptr;
 }
 
-void * ClickHouseDictionary_v1_loadKeys(void * data_ptr,
+void * ClickHouseDictionary_v2_loadKeys(void * data_ptr,
     ClickHouseLibrary::CStrings * settings,
     ClickHouseLibrary::CStrings * columns,
     const ClickHouseLibrary::VectorUInt64 * requested_rows)
@@ -148,13 +149,13 @@ void * ClickHouseDictionary_v1_loadKeys(void * data_ptr,
     return nullptr;
 }
 
-void * ClickHouseDictionary_v1_dataAllocate()
+void * ClickHouseDictionary_v2_dataAllocate()
 {
     auto data_ptr = new DataHolder;
     return data_ptr;
 }
 
-void ClickHouseDictionary_v1_dataDelete(void * data_ptr)
+void ClickHouseDictionary_v2_dataDelete(void * data_ptr)
 {
     auto ptr = static_cast<DataHolder *>(data_ptr);
     delete ptr;

libs/libcommon/include/common/iostream_debug_helpers.h

@@ -22,6 +22,7 @@
 #define DUMP3(V1, V2, V3) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3) << "\n";
 #define DUMP4(V1, V2, V3, V4) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3)<< ", " << DUMPS(V4) << "\n";
 #define DUMP5(V1, V2, V3, V4, V5) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3)<< ", " << DUMPS(V4) << ", " << DUMPS(V5) << "\n";
+#define DUMP6(V1, V2, V3, V4, V5, V6) DUMPHEAD << DUMPS(V1) << ", " << DUMPS(V2) << ", " << DUMPS(V3)<< ", " << DUMPS(V4) << ", " << DUMPS(V5) << ", " << DUMPS(V6) << "\n";
 
 
 namespace std