From 4c2820b9bbd6e827bdc68f5328447706ed03d816 Mon Sep 17 00:00:00 2001 From: haozi007 Date: Tue, 14 Jul 2020 11:24:09 +0800 Subject: [PATCH] lcr: support image tar split 1. add tar split entry json schema; 2. add crc64 function Signed-off-by: haozi007 --- CMakeLists.txt | 1 + src/json/schema/storage/entry.json | 27 +++++ tests/CMakeLists.txt | 1 + tests/go_crc64_ut.cpp | 164 +++++++++++++++++++++++++++++ third_party/go_crc64.c | 137 ++++++++++++++++++++++++ third_party/go_crc64.h | 54 ++++++++++ 6 files changed, 384 insertions(+) create mode 100644 src/json/schema/storage/entry.json create mode 100644 tests/go_crc64_ut.cpp create mode 100644 third_party/go_crc64.c create mode 100644 third_party/go_crc64.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 4aa9bb3..f2edb64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,7 @@ install(FILES src/lcrcontainer.h DESTINATION include/lcr) install(FILES third_party/log.h DESTINATION include/isula_libutils) +install(FILES third_party/go_crc64.h DESTINATION include/isula_libutils) install(FILES src/json/oci_runtime_hooks.h DESTINATION include/isula_libutils) install(FILES src/json/parse_common.h DESTINATION include/isula_libutils) install(DIRECTORY ${CMAKE_BINARY_DIR}/json/ DESTINATION include/isula_libutils diff --git a/src/json/schema/storage/entry.json b/src/json/schema/storage/entry.json new file mode 100644 index 0000000..3b7c9fb --- /dev/null +++ b/src/json/schema/storage/entry.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "type": { + "type": "int32" + }, + "name": { + "type": "string" + }, + "name_raw": { + "type": "array", + "items": { + "type": "byte" + } + }, + "size": { + "type": "int64" + }, + "payload": { + "type": "string" + }, + "position": { + "type": "int32" + } + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e2ad103..8110f10 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -70,6 +70,7 @@ endif() # api testcase _DEFINE_NEW_TEST(log_ut log_testcase) _DEFINE_NEW_TEST(libocispec_ut libocispec_testcase) +_DEFINE_NEW_TEST(go_crc64_ut go_crc64_testcase) # mock test for run lcov to generate html diff --git a/tests/go_crc64_ut.cpp b/tests/go_crc64_ut.cpp new file mode 100644 index 0000000..a62b5e8 --- /dev/null +++ b/tests/go_crc64_ut.cpp @@ -0,0 +1,164 @@ +/****************************************************************************** + * iSula-libutils: utils library for iSula + * + * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + * + * Authors: + * Haozi007 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ********************************************************************************/ +#include + +#include + +#include +#include + +#include "go_crc64.h" + +TEST(go_crc64_testcase, test_new_isula_crc_table) +{ + const isula_crc_table_t* ctab = new_isula_crc_table(ISO_POLY); + ASSERT_NE(ctab, nullptr); + ASSERT_EQ(ctab->inited, true); + + ctab = new_isula_crc_table(0x88888); + ASSERT_EQ(ctab, nullptr); +} + +TEST(go_crc64_testcase, test_isula_crc_update) +{ + const isula_crc_table_t* ctab = new_isula_crc_table(ISO_POLY); + ASSERT_NE(ctab, nullptr); + ASSERT_EQ(ctab->inited, true); + + uint64_t crc = 0; + bool ret; + + unsigned char buf1[] = "haozi"; + ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x39804c3418100000); + + crc = 0; + unsigned char buf2[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0xb8a4e0d5cfd9d3d6); + + crc = 0; + unsigned char buf3[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyy"; + ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x9fb11ff8b8a4e0d5); + + // check update with before crc + crc = 0; + ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x39804c3418100000); + + ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x8c862c945a1dbbfa); + + ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0xaff9fc095bb587c6); +} + +static bool check_sum(uint8_t sums[8], uint64_t crc) +{ + uint64_t tmp = 0; + size_t i; + + for (i = 0; i < 8; i++) { + tmp |= sums[i]; + if (i == 7) { + break; + } + tmp <<= 8; + } + + return (tmp == crc); +} + +TEST(go_crc64_testcase, test_isula_crc_sum) +{ + const isula_crc_table_t* ctab = new_isula_crc_table(ISO_POLY); + ASSERT_NE(ctab, nullptr); + ASSERT_EQ(ctab->inited, true); + + uint64_t crc = 0; + bool ret; + uint8_t sums[8] = {0}; + + unsigned char buf1[] = "haozi"; + ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x39804c3418100000); + + isula_crc_sum(crc, sums); + ret = check_sum(sums, crc); + ASSERT_EQ(ret, true); + + crc = 0; + unsigned char buf2[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0xb8a4e0d5cfd9d3d6); + + isula_crc_sum(crc, sums); + ret = check_sum(sums, crc); + ASSERT_EQ(ret, true); + + crc = 0; + unsigned char buf3[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyy"; + ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x9fb11ff8b8a4e0d5); + + isula_crc_sum(crc, sums); + ret = check_sum(sums, crc); + ASSERT_EQ(ret, true); + + // check update with before crc + crc = 0; + ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x39804c3418100000); + + isula_crc_sum(crc, sums); + ret = check_sum(sums, crc); + ASSERT_EQ(ret, true); + + ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0x8c862c945a1dbbfa); + + isula_crc_sum(crc, sums); + ret = check_sum(sums, crc); + ASSERT_EQ(ret, true); + + ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1); + ASSERT_EQ(ret, true); + ASSERT_EQ(crc, 0xaff9fc095bb587c6); + + isula_crc_sum(crc, sums); + ret = check_sum(sums, crc); + ASSERT_EQ(ret, true); +} + diff --git a/third_party/go_crc64.c b/third_party/go_crc64.c new file mode 100644 index 0000000..ef180d8 --- /dev/null +++ b/third_party/go_crc64.c @@ -0,0 +1,137 @@ +/****************************************************************************** + * ported from golang hash/crc64.go + * + * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + * + * Authors: + * Haozi007 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ********************************************************************************/ + +#include +#include + +#include "go_crc64.h" + +pthread_mutex_t g_crc_lock = PTHREAD_MUTEX_INITIALIZER; +isula_crc_table_t g_iso_crc_table; + +static void make_table(isula_crc_table_t *tab, uint64_t poly) +{ + uint64_t i, j; + uint64_t t; + + for (i = 0; i < CRC_COL_LEN; i++) { + t = i; + for (j = 0; j < CRC_ROW_LEN; j++) { + if ((t & 1) == 1) { + t = (t >> 1) ^ poly; + } else { + t >>= 1; + } + } + tab->table[0][i] = t; + } +} + +static void make_crc_table(isula_crc_table_t *tab) +{ + uint64_t i, j; + uint64_t crc; + + for (i = 0; i < CRC_COL_LEN; i++) { + crc = tab->table[0][i]; + for (j = 1; j < CRC_ROW_LEN; j++) { + crc = tab->table[0][(crc&0xff)] ^ (crc >> 8); + tab->table[j][i] = crc; + } + } +} + +const isula_crc_table_t* new_isula_crc_table(uint64_t poly) +{ + isula_crc_table_t *ret = NULL; + if (pthread_mutex_lock(&g_crc_lock) != 0) { + return ret; + } + + switch (poly) { + case ISO_POLY: + ret = &g_iso_crc_table; + break; + default: + goto out; + } + // ret must be non-null + if (ret->inited) { + goto out; + } + ret->inited = true; + make_table(ret, poly); + make_crc_table(ret); + +out: + (void)pthread_mutex_unlock(&g_crc_lock); + return ret; +} + +bool isula_crc_update(const isula_crc_table_t *tab, uint64_t *crc, unsigned char *data, size_t data_len) +{ + size_t i = 0; + uint64_t tcrc = 0; + + if (tab == NULL || !(tab->inited) || crc == NULL) { + // invalid table, just return origin crc + return false; + } + tcrc = ~(*crc); + + if (data_len >= 64) { + for (; i + CRC_ROW_LEN < data_len; i = i + CRC_ROW_LEN) { + tcrc ^= (uint64_t)data[i] | ((uint64_t)data[i+1] << 8) | ((uint64_t)data[i+2] << 16) | + ((uint64_t)data[i+3] << 24) | ((uint64_t)data[i+4] << 32) | ((uint64_t)data[i+5] << 40) | + ((uint64_t)data[i+6] << 48) | ((uint64_t)data[i+7] << 56); + + tcrc = tab->table[7][tcrc & 0xff] ^ + tab->table[6][(tcrc >> 8) & 0xff] ^ + tab->table[5][(tcrc >> 16) & 0xff] ^ + tab->table[4][(tcrc >> 24) & 0xff] ^ + tab->table[3][(tcrc >> 32) & 0xff] ^ + tab->table[2][(tcrc >> 40) & 0xff] ^ + tab->table[1][(tcrc >> 48) & 0xff] ^ + tab->table[0][(tcrc >> 56)]; + } + } + + for (; i < data_len; i++) { + tcrc = tab->table[0][((uint8_t)tcrc) ^ ((uint8_t)data[i])] ^ (tcrc >> 8); + } + + *crc = ~tcrc; + return true; +} + +void isula_crc_sum(uint64_t crc, uint8_t data[8]) +{ + data[0] = (uint8_t)(crc >> 56); + data[1] = (uint8_t)(crc >> 48); + data[2] = (uint8_t)(crc >> 40); + data[3] = (uint8_t)(crc >> 32); + data[4] = (uint8_t)(crc >> 24); + data[5] = (uint8_t)(crc >> 16); + data[6] = (uint8_t)(crc >> 8); + data[7] = (uint8_t)(crc); +} diff --git a/third_party/go_crc64.h b/third_party/go_crc64.h new file mode 100644 index 0000000..23433d5 --- /dev/null +++ b/third_party/go_crc64.h @@ -0,0 +1,54 @@ +/****************************************************************************** + * ported from golang hash/crc64.go + * + * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + * + * Authors: + * Haozi007 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ********************************************************************************/ + +#ifndef __CRC64_UTIL_H +#define __CRC64_UTIL_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define CRC_ROW_LEN 8 +#define CRC_COL_LEN 256 +#define ISO_POLY 0xD800000000000000 + +typedef struct __isula_crc_table_t { + bool inited; + uint64_t table[CRC_ROW_LEN][CRC_COL_LEN]; +} isula_crc_table_t; + +const isula_crc_table_t* new_isula_crc_table(uint64_t poly); + +bool isula_crc_update(const isula_crc_table_t *tab, uint64_t *crc, unsigned char *data, size_t data_len); + +void isula_crc_sum(uint64_t crc, uint8_t data[8]); + +#ifdef __cplusplus +} +#endif + +#endif -- GitLab