Merge pull request #6715 from yandex/remove-mimalloc

Remove mimalloc

.gitmodules

@@ -97,9 +97,6 @@
 [submodule "contrib/rapidjson"]
 	path = contrib/rapidjson
 	url = https://github.com/Tencent/rapidjson
-[submodule "contrib/mimalloc"]
-	path = contrib/mimalloc
-	url = https://github.com/ClickHouse-Extras/mimalloc
 [submodule "contrib/fastops"]
 	path = contrib/fastops
 	url = https://github.com/ClickHouse-Extras/fastops

CMakeLists.txt

@@ -340,7 +340,6 @@ include (cmake/find_consistent-hashing.cmake)
 include (cmake/find_base64.cmake)
 include (cmake/find_parquet.cmake)
 include (cmake/find_hyperscan.cmake)
-include (cmake/find_mimalloc.cmake)
 include (cmake/find_simdjson.cmake)
 include (cmake/find_rapidjson.cmake)
 include (cmake/find_fastops.cmake)

cmake/find_mimalloc.cmake

-if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT ARCH_PPC64LE)
-    option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" FALSE)
-endif ()
-
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h")
-    message (WARNING "submodule contrib/mimalloc is missing. to fix try run: \n git submodule update --init --recursive")
-    return()
-endif ()
-
-if (ENABLE_MIMALLOC)
-    message (FATAL_ERROR "Mimalloc is not production ready. (Disable with cmake -D ENABLE_MIMALLOC=0). If you want to use mimalloc, you must manually remove this message.")
-
-    set (MIMALLOC_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include)
-    set (USE_MIMALLOC 1)
-    set (MIMALLOC_LIBRARY mimalloc-static)
-    message (STATUS "Using mimalloc: ${MIMALLOC_INCLUDE_DIR} : ${MIMALLOC_LIBRARY}")
-endif ()

mimalloc @ a787bdeb

-Subproject commit a787bdebce94bf3776dc0d1ad597917f479ab8d5

dbms/CMakeLists.txt

@@ -256,11 +256,6 @@ if(RE2_INCLUDE_DIR)
     target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
 endif()
 
-if (USE_MIMALLOC)
-    target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MIMALLOC_INCLUDE_DIR})
-    target_link_libraries (clickhouse_common_io PRIVATE ${MIMALLOC_LIBRARY})
-endif ()
-
 if(CPUID_LIBRARY)
     target_link_libraries(clickhouse_common_io PRIVATE ${CPUID_LIBRARY})
 endif()

dbms/src/Common/MiAllocator.cpp

-#include "MiAllocator.h"
-
-#if USE_MIMALLOC
-#include <mimalloc.h>
-
-#include <Common/Exception.h>
-#include <Common/formatReadable.h>
-#include <IO/WriteHelpers.h>
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int CANNOT_ALLOCATE_MEMORY;
-}
-
-void * MiAllocator::alloc(size_t size, size_t alignment)
-{
-    void * ptr;
-    if (alignment == 0)
-    {
-        ptr = mi_malloc(size);
-        if (!ptr)
-            DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-    }
-    else
-    {
-        ptr = mi_malloc_aligned(size, alignment);
-        if (!ptr)
-            DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc (mi_malloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-    }
-    return ptr;
-}
-
-void MiAllocator::free(void * buf, size_t)
-{
-    mi_free(buf);
-}
-
-void * MiAllocator::realloc(void * old_ptr, size_t, size_t new_size, size_t alignment)
-{
-    if (old_ptr == nullptr)
-        return alloc(new_size, alignment);
-
-    if (new_size == 0)
-    {
-        mi_free(old_ptr);
-        return nullptr;
-    }
-
-    void * ptr;
-
-    if (alignment == 0)
-    {
-        ptr = mi_realloc(old_ptr, alignment);
-        if (!ptr)
-            DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-    }
-    else
-    {
-        ptr = mi_realloc_aligned(old_ptr, new_size, alignment);
-        if (!ptr)
-            DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc (mi_realloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-    }
-    return ptr;
-}
-
-}
-
-#endif

dbms/src/Common/MiAllocator.h

-#pragma once
-
-#include <Common/config.h>
-
-#if USE_MIMALLOC
-#include <cstddef>
-
-namespace DB
-{
-
-/*
- * This is a different allocator that is based on mimalloc (Microsoft malloc).
- * It can be used separately from main allocator to catch heap corruptions and vulnerabilities (for example, for caches).
- * We use MI_SECURE mode in mimalloc to achieve such behaviour.
- */
-struct MiAllocator
-{
-    static void * alloc(size_t size, size_t alignment = 0);
-
-    static void free(void * buf, size_t);
-
-    static void * realloc(void * old_ptr, size_t, size_t new_size, size_t alignment = 0);
-};
-
-}
-
-#endif

dbms/src/Common/config.h.in

@@ -8,6 +8,5 @@
 #cmakedefine01 USE_CPUID
 #cmakedefine01 USE_CPUINFO
 #cmakedefine01 USE_BROTLI
-#cmakedefine01 USE_MIMALLOC
 #cmakedefine01 USE_UNWIND
 #cmakedefine01 CLICKHOUSE_SPLIT_BINARY

dbms/src/Common/tests/CMakeLists.txt

@@ -76,8 +76,5 @@ target_link_libraries (cow_compositions PRIVATE clickhouse_common_io)
 add_executable (stopwatch stopwatch.cpp)
 target_link_libraries (stopwatch PRIVATE clickhouse_common_io)
 
-add_executable (mi_malloc_test mi_malloc_test.cpp)
-target_link_libraries (mi_malloc_test PRIVATE clickhouse_common_io)
-
 add_executable (symbol_index symbol_index.cpp)
 target_link_libraries (symbol_index PRIVATE clickhouse_common_io)

dbms/src/Common/tests/mi_malloc_test.cpp

-/** In addition to ClickHouse (Apache 2) license, this file can be also used under MIT license:
-
-MIT License
-
-Copyright (c) 2019 Yandex LLC, Alexey Milovidov
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-*/
-
-#include <map>
-#include <vector>
-#include <cstdint>
-#include <random>
-#include <stdexcept>
-#include <iostream>
-
-#include <Common/config.h>
-
-//#undef USE_MIMALLOC
-//#define USE_MIMALLOC 0
-
-#if USE_MIMALLOC
-
-#include <mimalloc.h>
-#define malloc mi_malloc
-#define free mi_free
-
-#else
-
-#include <stdlib.h>
-
-#endif
-
-
-size_t total_size{0};
-
-struct Allocation
-{
-    void * ptr = nullptr;
-    size_t size = 0;
-
-    Allocation() {}
-
-    Allocation(size_t size_)
-        : size(size_)
-    {
-        ptr = malloc(size);
-        if (!ptr)
-            throw std::runtime_error("Cannot allocate memory");
-        total_size += size;
-    }
-
-    ~Allocation()
-    {
-        if (ptr)
-        {
-            free(ptr);
-            total_size -= size;
-        }
-        ptr = nullptr;
-    }
-
-    Allocation(const Allocation &) = delete;
-
-    Allocation(Allocation && rhs)
-    {
-        ptr = rhs.ptr;
-        size = rhs.size;
-        rhs.ptr = nullptr;
-        rhs.size = 0;
-    }
-};
-
-
-int main(int, char **)
-{
-    std::vector<Allocation> allocations;
-
-    constexpr size_t limit = 100000000;
-    constexpr size_t min_alloc_size = 65536;
-    constexpr size_t max_alloc_size = 10000000;
-
-    std::mt19937 rng;
-    auto distribution = std::uniform_int_distribution(min_alloc_size, max_alloc_size);
-
-    size_t total_allocations = 0;
-
-    while (true)
-    {
-        size_t size = distribution(rng);
-
-        while (total_size + size > limit)
-            allocations.pop_back();
-
-        allocations.emplace_back(size);
-
-        ++total_allocations;
-        if (total_allocations % (1ULL << 20) == 0)
-            std::cerr << "Total allocations: " << total_allocations << "\n";
-    }
-}

dbms/src/DataStreams/MarkInCompressedFile.h

@@ -6,10 +6,6 @@
 #include <IO/WriteHelpers.h>
 #include <Common/PODArray.h>
 
-#include <Common/config.h>
-#if USE_MIMALLOC
-#include <Common/MiAllocator.h>
-#endif
 
 namespace DB
 {
@@ -43,9 +39,7 @@ struct MarkInCompressedFile
     }
 
 };
-#if USE_MIMALLOC
-using MarksInCompressedFile = PODArray<MarkInCompressedFile, 4096, MiAllocator>;
-#else
+
 using MarksInCompressedFile = PODArray<MarkInCompressedFile>;
-#endif
+
 }

dbms/src/IO/UncompressedCache.h

@@ -6,11 +6,6 @@
 #include <Common/ProfileEvents.h>
 #include <IO/BufferWithOwnMemory.h>
 
-#include <Common/config.h>
-#if USE_MIMALLOC
-#include <Common/MiAllocator.h>
-#endif
-
 
 namespace ProfileEvents
 {
@@ -25,11 +20,7 @@ namespace DB
 
 struct UncompressedCacheCell
 {
-#if USE_MIMALLOC
-    Memory<MiAllocator> data;
-#else
     Memory<> data;
-#endif
     size_t compressed_size;
     UInt32 additional_bytes;
 };