lcr: support image tar split

1. add tar split entry json schema;
2. add crc64 function
Signed-off-by: Nhaozi007 <liuhao27@huawei.com>

CMakeLists.txt

@@ -99,6 +99,7 @@ install(FILES src/lcrcontainer.h
 	DESTINATION include/lcr)
 
 install(FILES third_party/log.h DESTINATION include/isula_libutils)
+install(FILES third_party/go_crc64.h DESTINATION include/isula_libutils)
 install(FILES src/json/oci_runtime_hooks.h DESTINATION include/isula_libutils)
 install(FILES src/json/parse_common.h DESTINATION include/isula_libutils)
 install(DIRECTORY ${CMAKE_BINARY_DIR}/json/ DESTINATION include/isula_libutils

src/json/schema/storage/entry.json

+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "type": "object",
+    "properties": {
+        "type": {
+            "type": "int32"
+        },
+        "name": {
+            "type": "string"
+        },
+        "name_raw": {
+            "type": "array",
+            "items": {
+                "type": "byte"
+            }
+        },
+        "size": {
+            "type": "int64"
+        },
+        "payload": {
+            "type": "string"
+        },
+        "position": {
+            "type": "int32"
+        }
+    }
+}

tests/CMakeLists.txt

@@ -70,6 +70,7 @@ endif()
 #   api testcase
 _DEFINE_NEW_TEST(log_ut log_testcase)
 _DEFINE_NEW_TEST(libocispec_ut libocispec_testcase)
+_DEFINE_NEW_TEST(go_crc64_ut go_crc64_testcase)
 
 
 # mock test for run lcov to generate html

tests/go_crc64_ut.cpp

+/******************************************************************************
+ * iSula-libutils: utils library for iSula
+ *
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+ *
+ * Authors:
+ * Haozi007 <liuhao27@huawei.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ********************************************************************************/
+#include <gtest/gtest.h>
+
+#include <iostream>
+
+#include <string.h>
+#include <unistd.h>
+
+#include "go_crc64.h"
+
+TEST(go_crc64_testcase, test_new_isula_crc_table)
+{
+    const isula_crc_table_t* ctab = new_isula_crc_table(ISO_POLY);
+    ASSERT_NE(ctab, nullptr);
+    ASSERT_EQ(ctab->inited, true);
+
+    ctab = new_isula_crc_table(0x88888);
+    ASSERT_EQ(ctab, nullptr);
+}
+
+TEST(go_crc64_testcase, test_isula_crc_update)
+{
+    const isula_crc_table_t* ctab = new_isula_crc_table(ISO_POLY);
+    ASSERT_NE(ctab, nullptr);
+    ASSERT_EQ(ctab->inited, true);
+
+    uint64_t crc = 0;
+    bool ret;
+
+    unsigned char buf1[] = "haozi";
+    ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x39804c3418100000);
+
+    crc = 0;
+    unsigned char buf2[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+    ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0xb8a4e0d5cfd9d3d6);
+
+    crc = 0;
+    unsigned char buf3[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyy";
+    ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x9fb11ff8b8a4e0d5);
+
+    // check update with before crc
+    crc = 0;
+    ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x39804c3418100000);
+
+    ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x8c862c945a1dbbfa);
+
+    ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0xaff9fc095bb587c6);
+}
+
+static bool check_sum(uint8_t sums[8], uint64_t crc)
+{
+    uint64_t tmp = 0;
+    size_t i;
+
+    for (i = 0; i < 8; i++) {
+        tmp |= sums[i];
+        if (i == 7) {
+            break;
+        }
+        tmp <<= 8;
+    }
+
+    return (tmp == crc);
+}
+
+TEST(go_crc64_testcase, test_isula_crc_sum)
+{
+    const isula_crc_table_t* ctab = new_isula_crc_table(ISO_POLY);
+    ASSERT_NE(ctab, nullptr);
+    ASSERT_EQ(ctab->inited, true);
+
+    uint64_t crc = 0;
+    bool ret;
+    uint8_t sums[8] = {0};
+
+    unsigned char buf1[] = "haozi";
+    ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x39804c3418100000);
+
+    isula_crc_sum(crc, sums);
+    ret = check_sum(sums, crc);
+    ASSERT_EQ(ret, true);
+
+    crc = 0;
+    unsigned char buf2[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+    ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0xb8a4e0d5cfd9d3d6);
+
+    isula_crc_sum(crc, sums);
+    ret = check_sum(sums, crc);
+    ASSERT_EQ(ret, true);
+
+    crc = 0;
+    unsigned char buf3[] = "haozixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyy";
+    ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x9fb11ff8b8a4e0d5);
+
+    isula_crc_sum(crc, sums);
+    ret = check_sum(sums, crc);
+    ASSERT_EQ(ret, true);
+
+    // check update with before crc
+    crc = 0;
+    ret = isula_crc_update(ctab, &crc, buf1, sizeof(buf1) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x39804c3418100000);
+
+    isula_crc_sum(crc, sums);
+    ret = check_sum(sums, crc);
+    ASSERT_EQ(ret, true);
+
+    ret = isula_crc_update(ctab, &crc, buf2, sizeof(buf2) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0x8c862c945a1dbbfa);
+
+    isula_crc_sum(crc, sums);
+    ret = check_sum(sums, crc);
+    ASSERT_EQ(ret, true);
+
+    ret = isula_crc_update(ctab, &crc, buf3, sizeof(buf3) - 1);
+    ASSERT_EQ(ret, true);
+    ASSERT_EQ(crc, 0xaff9fc095bb587c6);
+
+    isula_crc_sum(crc, sums);
+    ret = check_sum(sums, crc);
+    ASSERT_EQ(ret, true);
+}
+

third_party/go_crc64.c

+/******************************************************************************
+ * ported from golang hash/crc64.go
+ *
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+ *
+ * Authors:
+ * Haozi007 <liuhao27@huawei.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ********************************************************************************/
+
+#include <stdlib.h>
+#include <pthread.h>
+
+#include "go_crc64.h"
+
+pthread_mutex_t g_crc_lock = PTHREAD_MUTEX_INITIALIZER;
+isula_crc_table_t g_iso_crc_table;
+
+static void make_table(isula_crc_table_t *tab, uint64_t poly)
+{
+    uint64_t i, j;
+    uint64_t t;
+
+    for (i = 0; i < CRC_COL_LEN; i++) {
+        t = i;
+        for (j = 0; j < CRC_ROW_LEN; j++) {
+            if ((t & 1) == 1) {
+                t = (t >> 1) ^ poly;
+            } else {
+                t >>= 1;
+            }
+        }
+        tab->table[0][i] = t;
+    }
+}
+
+static void make_crc_table(isula_crc_table_t *tab)
+{
+    uint64_t i, j;
+    uint64_t crc;
+
+    for (i = 0; i < CRC_COL_LEN; i++) {
+        crc = tab->table[0][i];
+        for (j = 1; j < CRC_ROW_LEN; j++) {
+            crc = tab->table[0][(crc&0xff)] ^ (crc >> 8);
+            tab->table[j][i] = crc;
+        }
+    }
+}
+
+const isula_crc_table_t* new_isula_crc_table(uint64_t poly)
+{
+    isula_crc_table_t *ret = NULL;
+    if (pthread_mutex_lock(&g_crc_lock) != 0) {
+        return ret;
+    }
+
+    switch (poly) {
+        case ISO_POLY:
+            ret = &g_iso_crc_table;
+            break;
+        default:
+            goto out;
+    }
+    // ret must be non-null
+    if (ret->inited) {
+        goto out;
+    }
+    ret->inited = true;
+    make_table(ret, poly);
+    make_crc_table(ret);
+
+out:
+    (void)pthread_mutex_unlock(&g_crc_lock);
+    return ret;
+}
+
+bool isula_crc_update(const isula_crc_table_t *tab, uint64_t *crc, unsigned char *data, size_t data_len)
+{
+    size_t i = 0;
+    uint64_t tcrc = 0;
+
+    if (tab == NULL || !(tab->inited) || crc == NULL) {
+        // invalid table, just return origin crc
+        return false;
+    }
+    tcrc = ~(*crc);
+
+    if (data_len >= 64) {
+        for (; i + CRC_ROW_LEN < data_len; i = i + CRC_ROW_LEN) {
+            tcrc ^= (uint64_t)data[i] | ((uint64_t)data[i+1] << 8) | ((uint64_t)data[i+2] << 16) |
+                ((uint64_t)data[i+3] << 24) | ((uint64_t)data[i+4] << 32) | ((uint64_t)data[i+5] << 40) |
+                ((uint64_t)data[i+6] << 48) | ((uint64_t)data[i+7] << 56);
+
+            tcrc = tab->table[7][tcrc & 0xff] ^
+                tab->table[6][(tcrc >> 8) & 0xff] ^
+                tab->table[5][(tcrc >> 16) & 0xff] ^
+                tab->table[4][(tcrc >> 24) & 0xff] ^
+                tab->table[3][(tcrc >> 32) & 0xff] ^
+                tab->table[2][(tcrc >> 40) & 0xff] ^
+                tab->table[1][(tcrc >> 48) & 0xff] ^
+                tab->table[0][(tcrc >> 56)];
+        }
+    }
+
+    for (; i < data_len; i++) {
+        tcrc = tab->table[0][((uint8_t)tcrc) ^ ((uint8_t)data[i])] ^ (tcrc >> 8);
+    }
+
+    *crc = ~tcrc;
+    return true;
+}
+
+void isula_crc_sum(uint64_t crc, uint8_t data[8])
+{
+    data[0] = (uint8_t)(crc >> 56);
+    data[1] = (uint8_t)(crc >> 48);
+    data[2] = (uint8_t)(crc >> 40);
+    data[3] = (uint8_t)(crc >> 32);
+    data[4] = (uint8_t)(crc >> 24);
+    data[5] = (uint8_t)(crc >> 16);
+    data[6] = (uint8_t)(crc >> 8);
+    data[7] = (uint8_t)(crc);
+}

third_party/go_crc64.h

+/******************************************************************************
+ * ported from golang hash/crc64.go
+ *
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+ *
+ * Authors:
+ * Haozi007 <liuhao27@huawei.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ********************************************************************************/
+
+#ifndef __CRC64_UTIL_H
+#define __CRC64_UTIL_H
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CRC_ROW_LEN 8
+#define CRC_COL_LEN 256
+#define ISO_POLY 0xD800000000000000
+
+typedef struct __isula_crc_table_t {
+    bool inited;
+    uint64_t table[CRC_ROW_LEN][CRC_COL_LEN];
+} isula_crc_table_t;
+
+const isula_crc_table_t* new_isula_crc_table(uint64_t poly);
+
+bool isula_crc_update(const isula_crc_table_t *tab, uint64_t *crc, unsigned char *data, size_t data_len);
+
+void isula_crc_sum(uint64_t crc, uint8_t data[8]);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif