Merge branch 'crc32-rebase' into 'master'

Crc32 rebase



See merge request !1466148

CMakeLists.txt

@@ -7,6 +7,9 @@ endif()
 
 add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:_GLIBCXX_USE_CXX11_ABI=0>)
 
+option(HIDDEN_DMB_SYMBOLS "hidden dmb symbols" FALSE)
+mark_as_advanced(HIDDEN_DMB_SYMBOLS)
+
 add_subdirectory(src)
 
 # Check if oblogmsg is being used directly or via add_subdirectory

include/LogRecord.h

@@ -21,7 +21,8 @@ namespace logmessage {
 
 class StrArray;
 struct LogRecInfo;
-
+extern bool LOGREC_CRC;
+extern bool LOGREC_PARSE_CRC;
 #define BR_FAKE_DDL_COLNAME "ddl"
 typedef enum RecordType {
   EINSERT = 0,

src/CMakeLists.txt

@@ -4,6 +4,7 @@ set(SRC_LIST LogMsgBuf.cpp
              MetaInfo.cpp
              MsgType.cpp
              MsgVarArea.cpp
+             Crc32.cpp
 )
 
 add_library(oblogmsg_shared SHARED ${SRC_LIST})
@@ -11,6 +12,8 @@ add_library(oblogmsg_static STATIC ${SRC_LIST})
 
 foreach(T IN ITEMS oblogmsg_shared oblogmsg_static)
   target_compile_features(${T} PUBLIC cxx_std_11)
+  target_compile_options(${T} PRIVATE -Wno-invalid-offsetof)
+
   set_target_properties(${T} PROPERTIES OUTPUT_NAME oblogmsg
                                         CXX_STANDARD 11
                                         CXX_EXTENSIONS OFF
@@ -18,7 +21,7 @@ foreach(T IN ITEMS oblogmsg_shared oblogmsg_static)
   target_include_directories(${T} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
                                   PUBLIC  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>)
 endforeach()
-
+target_compile_definitions(oblogmsg_static PRIVATE $<$<BOOL:${HIDDEN_DMB_SYMBOLS}>:_HIDDEN_DMB_SYMBOLS>)
 # ensure that the build results can be run on systems with lower libstdc++ version than the build system
 target_link_libraries(oblogmsg_shared PRIVATE -static-libstdc++)
 

src/Crc32.h

+// //////////////////////////////////////////////////////////
+// Crc32.cpp
+// Copyright (c) 2011-2015 Stephan Brumme. All rights reserved.
+// Slicing-by-16 contributed by Bulat Ziganshin
+// see http://create.stephan-brumme.com/disclaimer.html
+//
+
+// g++ -o Crc32 Crc32.cpp -O3 -lrt -march=native -mtune=native
+
+// if running on an embedded system, you might consider shrinking the
+// big Crc32Lookup table:
+// - crc32_bitwise doesn't need it at all
+// - crc32_halfbyte has its own small lookup table
+// - crc32_1byte    needs only Crc32Lookup[0]
+// - crc32_4bytes   needs only Crc32Lookup[0..3]
+// - crc32_8bytes   needs only Crc32Lookup[0..7]
+// - crc32_4x8bytes needs only Crc32Lookup[0..7]
+// - crc32_16bytes  needs all of Crc32Lookup
+uint32_t crc32_halfbyte(const void* data, size_t length, uint32_t previousCrc32 = 0);
+uint32_t crc32_1byte(const void* data, size_t length, uint32_t previousCrc32 = 0);
+uint32_t crc32_4bytes(const void* data, size_t length, uint32_t previousCrc32 = 0);
+uint32_t crc32_8bytes(const void* data, size_t length, uint32_t previousCrc32 = 0);
+uint32_t crc32_4x8bytes(const void* data, size_t length, uint32_t previousCrc32 = 0);
+uint32_t crc32_16bytes(const void* data, size_t length, uint32_t previousCrc32 = 0);
+uint32_t crc32_16bytes_prefetch(
+    const void* data, size_t length, uint32_t previousCrc32 = 0, size_t prefetchAhead = 256);
+#define crc32_fast crc32_16bytes

src/LogMsgBuf.cpp

@@ -19,6 +19,13 @@ See the Mulan PSL v2 for more details. */
 namespace oceanbase {
 namespace logmessage {
 
+
+#ifdef _HIDDEN_DMB_SYMBOLS
+  #define VISIBILITY_HIDDEN __attribute__((visibility("hidden")))
+#else
+  #define VISIBILITY_HIDDEN
+#endif
+
 typedef struct _LogMsgBuf {
   char* buf;
   size_t bufSize;
@@ -47,7 +54,7 @@ static inline void lm_check_buf(LogMsgBuf* lmbuf, size_t size, char*& pos, STRLE
   }
 }
 
-int LogMsgLocalInit()
+VISIBILITY_HIDDEN int LogMsgLocalInit()
 {
   LogMsgLocalDestroy();
   if (NULL == (lmb = new LogMsgBuf))
@@ -62,7 +69,7 @@ int LogMsgLocalInit()
   return 0;
 }
 
-void LogMsgLocalDestroy()
+VISIBILITY_HIDDEN void LogMsgLocalDestroy()
 {
   if (lmb == &lmb_global)
     return;
@@ -77,7 +84,7 @@ void LogMsgLocalDestroy()
   }
 }
 
-int LogMsgInit()
+VISIBILITY_HIDDEN int LogMsgInit()
 {
   if (lmb != &lmb_global) {
     LogMsgLocalDestroy();
@@ -91,7 +98,7 @@ int LogMsgInit()
   return 0;
 }
 
-void LogMsgDestroy()
+VISIBILITY_HIDDEN void LogMsgDestroy()
 {
   if (lmb != &lmb_global)
     return;
@@ -102,12 +109,12 @@ void LogMsgDestroy()
   lmb->buf = lmb->defaultBuf = NULL;
 }
 
-const char* LogMsgGetValueByOffset(size_t offset)
+VISIBILITY_HIDDEN const char* LogMsgGetValueByOffset(size_t offset)
 {
   return lmb->buf + offset + LogMsgHeadSize + sizeof(DT_TYPE) + sizeof(COUNT_TYPE);
 }
 
-size_t LogMsgAppendString(const char* string, size_t size)
+VISIBILITY_HIDDEN size_t LogMsgAppendString(const char* string, size_t size)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize;
   *(DT_TYPE*)(lmb->buf + lmb->bufPos) = (DT_TYPE)(DT_STRING);
@@ -122,7 +129,7 @@ size_t LogMsgAppendString(const char* string, size_t size)
   lmb->buf[lmb->bufPos - 1] = 0;
   return offset;
 }
-size_t LogMsgAppendString(const std::string& string)
+VISIBILITY_HIDDEN size_t LogMsgAppendString(const std::string& string)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize, size = string.size();
   *(DT_TYPE*)(lmb->buf + lmb->bufPos) = (DT_TYPE)(DT_STRING);
@@ -136,14 +143,14 @@ size_t LogMsgAppendString(const std::string& string)
   lmb->bufPos += (sizeof(DT_TYPE) + sizeof(STRLEN_TYPE) + size + 1);
   return offset;
 }
-size_t LogMsgAppendBuf(const char* data, size_t size)
+VISIBILITY_HIDDEN size_t LogMsgAppendBuf(const char* data, size_t size)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize;
   memcpy(lmb->buf + lmb->bufPos, data, size);
   lmb->bufPos += size;
   return offset;
 }
-size_t LogMsgAppendBuf(const std::string& string)
+VISIBILITY_HIDDEN size_t LogMsgAppendBuf(const std::string& string)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize;
   memcpy(lmb->buf + lmb->bufPos, string.c_str(), string.size());
@@ -151,7 +158,7 @@ size_t LogMsgAppendBuf(const std::string& string)
   return offset;
 }
 
-size_t LogMsgAppendBuf(const BinLogBuf* sa, size_t size)
+VISIBILITY_HIDDEN size_t LogMsgAppendBuf(const BinLogBuf* sa, size_t size)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize;
   *(DT_TYPE*)(lmb->buf + lmb->bufPos) = (DT_TYPE)(DT_STRING | DC_ARRAY);
@@ -185,7 +192,7 @@ size_t LogMsgAppendBuf(const BinLogBuf* sa, size_t size)
   lmb->bufPos = pos - lmb->buf;
   return offset;
 }
-size_t LogMsgAppendStringArray(std::vector<std::string*>& sa)
+VISIBILITY_HIDDEN size_t LogMsgAppendStringArray(std::vector<std::string*>& sa)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize, size = sa.size();
   *(DT_TYPE*)(lmb->buf + lmb->bufPos) = (DT_TYPE)(DT_STRING | DC_ARRAY);
@@ -219,7 +226,7 @@ size_t LogMsgAppendStringArray(std::vector<std::string*>& sa)
   lmb->bufPos = pos - lmb->buf;
   return offset;
 }
-size_t LogMsgAppendStringArray(const char** sa, size_t size)
+VISIBILITY_HIDDEN size_t LogMsgAppendStringArray(const char** sa, size_t size)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize;
   *(DT_TYPE*)(lmb->buf + lmb->bufPos) = (DT_TYPE)(DT_STRING | DC_ARRAY);
@@ -252,7 +259,7 @@ size_t LogMsgAppendStringArray(const char** sa, size_t size)
   lmb->bufPos = pos - lmb->buf;
   return offset;
 }
-size_t LogMsgAppendDataArray(std::vector<long>& sa)
+VISIBILITY_HIDDEN size_t LogMsgAppendDataArray(std::vector<long>& sa)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize;
   *(DT_TYPE*)(lmb->buf + lmb->bufPos) = (DT_TYPE)(DT_INT64 | DC_ARRAY);
@@ -276,7 +283,7 @@ size_t LogMsgAppendDataArray(std::vector<long>& sa)
   return offset;
 }
 
-size_t LogMsgAppendDataArray(uint8_t* sa, size_t size)
+VISIBILITY_HIDDEN size_t LogMsgAppendDataArray(uint8_t* sa, size_t size)
 {
   size_t offset = lmb->bufPos - LogMsgHeadSize;
   *(DT_TYPE*)(lmb->buf + lmb->bufPos) = (DT_TYPE)(DT_UINT8 | DC_ARRAY);
@@ -292,7 +299,7 @@ size_t LogMsgAppendDataArray(uint8_t* sa, size_t size)
   lmb->bufPos += size;
   return offset;
 }
-void LogMsgSetHead(size_t size)
+VISIBILITY_HIDDEN void LogMsgSetHead(size_t size)
 {
   lmb->bufPos = size + LogMsgHeadSize + sizeof(DT_TYPE) + sizeof(COUNT_TYPE);
   if (lmb->buf != lmb->defaultBuf) {
@@ -308,7 +315,7 @@ void LogMsgSetHead(size_t size)
     }
   }
 }
-void LogMsgCopyHead(const char* head, size_t size)
+VISIBILITY_HIDDEN void LogMsgCopyHead(const char* head, size_t size)
 {
   ((struct MsgHeader*)lmb->buf)->m_msgType = MT_VAR;
   toLeEndian(&(((struct MsgHeader*)lmb->buf)->m_msgType), sizeof(uint16_t));
@@ -326,13 +333,13 @@ void LogMsgCopyHead(const char* head, size_t size)
   memcpy(lmb->buf + LogMsgHeadSize + sizeof(DT_TYPE) + sizeof(COUNT_TYPE), head, size);
 }
 
-void LogMsgFroceSetHeadSize(size_t size)
+VISIBILITY_HIDDEN void LogMsgFroceSetHeadSize(size_t size)
 {
   *(COUNT_TYPE*)(lmb->buf + LogMsgHeadSize + sizeof(DT_TYPE)) = size;
   toLeEndian(lmb->buf + LogMsgHeadSize + sizeof(DT_TYPE), sizeof(COUNT_TYPE));
 }
 
-const char* LogMsgGetString(size_t* size)
+VISIBILITY_HIDDEN const char* LogMsgGetString(size_t* size)
 {
   *size = lmb->bufPos;
   if (lmb->buf != lmb->defaultBuf && lmb->bufPos > DefaultLogMsgBufSize)

src/LogRecord.cpp

@@ -20,6 +20,7 @@ See the Mulan PSL v2 for more details. */
 #include <cinttypes>
 #include <iostream>
 #include "LogMsgBuf.h"
+#include "Crc32.h"
 
 namespace oceanbase {
 namespace logmessage {
@@ -32,6 +33,8 @@ const uint16_t LOGREC_VERSION = 3;                       // version num
 const uint64_t LOGREC_SUB_VERSION = 0x0200000000000000;  // sub version num
 #define GET_LOGREC_SUB_VERSION(v) ((v)&0xFF00000000000000)
 const uint64_t LOGREC_INVALID_ID = 0;
+bool LOGREC_CRC = false;
+bool LOGREC_PARSE_CRC = false;
 
 // LogRecord head info
 struct PosOfLogMsg_vb {
@@ -511,6 +514,10 @@ struct LogRecInfo {
 
     exchangeEndInfoToLe(m_endInfo, count);
     exchangePosInfoToLe((const char*)m_posInfo, count);
+    if (m_tailParseOK && LOGREC_PARSE_CRC && count >= sizeof(struct EndOfLogMsg_v2) && m_endInfo->m_crc != 0) {
+      if (m_endInfo->m_crc != crc32_fast((char*)ptr, (const char*)v - (const char*)ptr + offsetof(EndOfLogMsg, m_crc)))
+        return -5;
+    }
     m_parsedOK = true;
     return 0;
   }
@@ -1385,7 +1392,10 @@ struct LogRecInfo {
       /*force set header length to PosOfLogMsg_v3，because client will check it*/
       *((uint32_t*)v - 1) = toLeEndianByType((uint32_t)sizeof(PosOfLogMsg_v3));
       delete[] posInfoToLe;
-
+      if (LOGREC_CRC) {
+        EndOfLogMsg* tail = (EndOfLogMsg*)(m.c_str() + m.size() - sizeof(EndOfLogMsg));
+        tail->m_crc = toLeEndianByType((uint32_t)crc32_fast(m.c_str(), (char*)tail - m.c_str() + offsetof(EndOfLogMsg, m_crc)));
+      }
       m_parsedOK = true;  // to support fetching
       *size = m.size();
       return m.c_str();
@@ -1426,7 +1436,10 @@ struct LogRecInfo {
       delete[] posInfoToLe;
 
       const char* msg = LogMsgGetString(size);
-
+      if (LOGREC_CRC) {
+        EndOfLogMsg* tail = (EndOfLogMsg*)LogMsgGetValueByOffset(((PosOfLogMsg_vc*)m_posInfo)->m_posOfEndInfo);
+        tail->m_crc = toLeEndianByType((uint32_t)crc32_fast(msg, (char*)tail - msg + offsetof(EndOfLogMsg, m_crc)));
+      }
       /* Serialize the header */
       if (reserveMemory) {
         m_reservedMemory = true;