diff --git a/src/daemon/modules/image/oci/registry/auths.c b/src/daemon/modules/image/oci/registry/auths.c index ce05bbdab0d60fcaca09c4702c2e1ed9c09851b3..761a2e1998ba4e879ad1681babee459bd46cc936 100644 --- a/src/daemon/modules/image/oci/registry/auths.c +++ b/src/daemon/modules/image/oci/registry/auths.c @@ -63,7 +63,7 @@ void auths_set_dir(char *auth_dir) return; } -static int decode_auth(char *encoded, char **username, char **password) +static int decode_auth_aes(char *encoded, char **username, char **password) { int nret = 0; int ret = 0; @@ -147,7 +147,7 @@ out: return ret; } -static char *encode_auth(char *username, char *password) +static char *encode_auth_aes(char *username, char *password) { int ret = 0; int nret = 0; @@ -259,9 +259,9 @@ int auths_load(char *host, char **username, char **password) for (i = 0; i < auths->auths->len; i++) { if (!strcmp(host, auths->auths->keys[i])) { - ret = decode_auth(auths->auths->values[i]->auth, username, password); + ret = decode_auth_aes(auths->auths->values[i]->auth, username, password); if (ret != 0) { - ERROR("Decode auth failed"); + ERROR("Decode auth with aes failed"); goto out; } } @@ -426,9 +426,9 @@ int auths_save(char *host, char *username, char *password) } } - auth = encode_auth(username, password); + auth = encode_auth_aes(username, password); if (auth == NULL) { - ERROR("encode auth failed"); + ERROR("encode auth with aes failed"); ret = -1; goto out; } diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c index 18cdf2d75a06a01e1f5e95e458123b5ad18f9810..8b6cc0bf3d803a9eb9aed8beda5cd4aa909f7b23 100644 --- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c @@ -25,9 +25,14 @@ #include #include #include +#include +#include +#include #include "storage.h" #include "isula_libutils/json_common.h" +#include "isula_libutils/storage_entry.h" +#include "isula_libutils/go_crc64.h" #include "layer.h" #include "driver.h" #include "linked_list.h" @@ -36,9 +41,13 @@ #include "utils.h" #include "utils_array.h" #include "utils_file.h" +#include "util_gzip.h" #include "isula_libutils/log.h" #include "constants.h" +#define PAYLOAD_CRC_LEN 12 +#define FILE_CRC_LEN 8 + struct io_read_wrapper; typedef struct __layer_store_metadata_t { @@ -56,6 +65,11 @@ typedef struct digest_layer { size_t layer_list_len; } digest_layer_t; +typedef struct { + FILE *tmp_file; + storage_entry *entry; +} tar_split; + static layer_store_metadata g_metadata; static char *g_root_dir; static char *g_run_dir; @@ -1656,3 +1670,307 @@ void layer_store_exit() { graphdriver_cleanup(); } + +static uint64_t payload_to_crc(char *payload) +{ + int ret = 0; + int i = 0; + uint64_t crc = 0; + uint8_t crc_sums[FILE_CRC_LEN] = {0}; + + ret = util_base64_decode(payload, PAYLOAD_CRC_LEN, crc_sums, FILE_CRC_LEN); + if (ret < 0) { + ERROR("decode tar split payload from base64 failed, payload %s", payload); + return -1; + } + + for (i = 0; i < FILE_CRC_LEN; i++) { + crc |= crc_sums[i]; + if (i == FILE_CRC_LEN - 1) { + break; + } + crc <<= FILE_CRC_LEN; + } + + return crc; +} + +static int file_crc(char *file, uint64_t *crc, uint64_t policy) +{ +#define BLKSIZE 32768 + int ret = 0; + const isula_crc_table_t * ctab = NULL; + int fd = 0; + void *buffer = NULL; + ssize_t size = 0; + + fd = util_open(file, O_RDONLY, 0); + if (fd < 0) { + ERROR("Open file: %s, failed: %s", file, strerror(errno)); + return -1; + } + + ctab = new_isula_crc_table(policy); + if (ctab == NULL || !ctab->inited) { + ERROR("create crc table failed"); + ret = -1; + goto out; + } + + buffer = util_common_calloc_s(BLKSIZE); + if (buffer == NULL) { + ERROR("out of memory"); + ret = -1; + goto out; + } + + *crc = 0; + while (true) { + size = util_read_nointr(fd, buffer, BLKSIZE); + if (size < 0) { + ERROR("read file %s failed: %s", file, strerror(errno)); + ret = -1; + break; + } else if (size == 0) { + break; + } + + if (!isula_crc_update(ctab, crc, buffer, (size_t)size)) { + ERROR("crc update failed"); + ret = -1; + break; + } + } + +out: + + close(fd); + free(buffer); + + return ret; +} + +static int valid_crc(storage_entry *entry, char *rootfs) +{ + int ret = 0; + int nret = 0; + uint64_t crc = 0; + uint64_t expected_crc = 0; + char file[PATH_MAX] = {0}; + struct stat st; + + nret = snprintf(file, PATH_MAX, "%s/%s", rootfs, entry->name); + if (nret < 0 || nret >= PATH_MAX) { + ERROR("snprintf %s/%s failed", rootfs, entry->name); + ret = -1; + goto out; + } + + if (entry->payload == NULL) { + if (stat(file, &st) != 0) { + ERROR("stat file %s failed: %s", file, strerror(errno)); + ret = -1; + goto out; + } + } else { + if (strlen(entry->payload) != PAYLOAD_CRC_LEN) { + ERROR("invalid payload %s of file %s", entry->payload, file); + ret = -1; + goto out; + } + + ret = file_crc(file, &crc, ISO_POLY); + if (ret != 0) { + ERROR("calc crc of file %s failed", file); + ret = -1; + goto out; + } + + expected_crc = payload_to_crc(entry->payload); + if (crc != expected_crc) { + ERROR("file %s crc 0x%jx not as expected 0x%jx", file, crc, expected_crc); + ret = -1; + goto out; + } + } + +out: + + return ret; +} + +static void free_tar_split(tar_split *ts) +{ + if (ts == NULL) { + return; + } + free_storage_entry(ts->entry); + ts->entry = NULL; + if (ts->tmp_file != NULL) { + fclose(ts->tmp_file); + ts->tmp_file = NULL; + } + free(ts); + return; +} + +static tar_split *new_tar_split(layer_t *l) +{ + int ret = 0; + tar_split *ts = NULL; + char *tspath = NULL; + + ts = util_common_calloc_s(sizeof(tar_split)); + if (ts == NULL) { + ERROR("out of memory"); + return NULL; + } + + ts->tmp_file = tmpfile(); + if (ts->tmp_file == NULL) { + ERROR("create tmpfile failed: %s", strerror(errno)); + ret = -1; + goto out; + } + + tspath = tar_split_path(l->slayer->id); + if (tspath == NULL) { + ERROR("get tar split path of layer %s failed", l->slayer->id); + ret = -1; + goto out; + } + + ret = util_gzip_d(tspath, ts->tmp_file); + if (ret != 0) { + ERROR("unzip tar split file %s failed", tspath); + goto out; + } + + rewind(ts->tmp_file); + +out: + if (ret != 0) { + free_tar_split(ts); + ts = NULL; + } + free(tspath); + + return ts; +} + +static int next_tar_split_entry(tar_split *ts, storage_entry **entry) +{ + int ret = 0; + int nret = 0; + char *pline = NULL; + size_t length = 0; + char *errmsg = NULL; + + nret = getline(&pline, &length, ts->tmp_file); + if (nret == -1) { + // end of file + if (errno == 0) { + *entry = NULL; + return 0; + } + ERROR("error read line from tar split: %s", strerror(errno)); + ret = -1; + goto out; + } + + util_trim_newline(pline); + + if (ts->entry != NULL) { + free_storage_entry(ts->entry); + } + ts->entry = storage_entry_parse_data(pline, NULL, &errmsg); + if (ts->entry == NULL) { + ERROR("parse tar split entry failed: %s\nline:%s", errmsg, pline); + ret = -1; + goto out; + } + + *entry = ts->entry; + +out: + free(errmsg); + free(pline); + + return ret; +} + +static int do_integration_check(layer_t *l, char *rootfs) +{ +#define STORAGE_ENTRY_TYPE_CRC 1 + int ret = 0; + tar_split *ts = NULL; + storage_entry *entry = NULL; + + ts = new_tar_split(l); + if (ts == NULL) { + ERROR("new tar split for layer %s failed", l->slayer->id); + return -1; + } + + ret = next_tar_split_entry(ts, &entry); + if (ret != 0) { + ERROR("get next tar split entry failed"); + goto out; + } + while (entry != NULL) { + if (entry->type == STORAGE_ENTRY_TYPE_CRC) { + ret = valid_crc(entry, rootfs); + if (ret != 0) { + ERROR("integration check failed, layer %s, file %s", l->slayer->id, entry->name); + goto out; + } + } + + ret = next_tar_split_entry(ts, &entry); + if (ret != 0) { + ERROR("get next tar split entry failed"); + goto out; + } + } + +out: + free_tar_split(ts); + + return ret; +} + +int layer_store_check(const char *id) +{ + int ret = 0; + char *rootfs = NULL; + + layer_t *l = lookup_with_lock(id); + if (l == NULL || l->slayer == NULL) { + ERROR("layer %s not found when checking integration", id); + return -1; + } + + // It's a container layer, not a layer of image, ignore checking + if (l->slayer->diff_digest == NULL) { + goto out; + } + + rootfs = layer_store_mount(id, NULL); + if (rootfs == NULL) { + ERROR("mount layer of %s failed", id); + ret = -1; + goto out; + } + + ret = do_integration_check(l, rootfs); + if (ret != 0) { + ERROR("do integration check failed for layer %s", l->slayer->id); + goto out; + } + +out: + (void)layer_store_umount(id, false); + layer_ref_dec(l); + + return ret; +} diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h index 34a607725ffa3a8b5a6fbb268906d2240b877a68..24db32098958a12eb22a3f7340054ae0a3615a45 100644 --- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h @@ -72,6 +72,8 @@ void free_layer_opts(struct layer_opts *opts); int layer_store_get_layer_fs_info(const char *layer_id, imagetool_fs_info *fs_info); +int layer_store_check(const char *id); + #ifdef __cplusplus } #endif diff --git a/src/utils/tar/util_gzip.c b/src/utils/tar/util_gzip.c index 3688af478439a42badf0fc39b594c5a12cb30332..c4cdd4d72c29275ab3b9ce2bb2b615525f22761a 100644 --- a/src/utils/tar/util_gzip.c +++ b/src/utils/tar/util_gzip.c @@ -42,7 +42,7 @@ int util_gzip_z(const char *srcfile, const char *dstfile, const mode_t mode) stream = gzopen(dstfile, "w"); if (stream == NULL) { - ERROR("gzopen error: %s", strerror(errno)); + ERROR("gzopen %s error: %s", dstfile, strerror(errno)); close(srcfd); return -1; } @@ -89,27 +89,19 @@ out: } // Decompress -int util_gzip_d(const char *srcfile, const char *dstfile, const mode_t mode) +int util_gzip_d(const char *srcfile, const FILE *dstfp) { gzFile stream = NULL; - int dstfd = 0; const char *gzerr = NULL; int errnum = 0; int ret = 0; - ssize_t size = 0; + size_t size = 0; void *buffer = NULL; size_t n = 0; stream = gzopen(srcfile, "r"); if (stream == NULL) { - ERROR("gzopen error: %s", strerror(errno)); - return -1; - } - - dstfd = util_open(dstfile, O_WRONLY | O_CREAT | O_TRUNC, mode); - if (dstfd < 0) { - ERROR("Creat file: %s failed: %s", dstfile, strerror(errno)); - gzclose(stream); + ERROR("gzopen %s failed: %s", srcfile, strerror(errno)); return -1; } @@ -132,8 +124,8 @@ int util_gzip_d(const char *srcfile, const char *dstfile, const mode_t mode) } if (n > 0) { - size = util_write_nointr(dstfd, buffer, n); - if (size < 0 || ((size_t)size) != n) { + size = fwrite(buffer, 1, n, (FILE *)dstfp); + if (size != n) { ret = -1; ERROR("Write file failed: %s", strerror(errno)); break; @@ -147,12 +139,9 @@ int util_gzip_d(const char *srcfile, const char *dstfile, const mode_t mode) out: gzclose(stream); - close(dstfd); free(buffer); - if (ret != 0) { - if (util_path_remove(dstfile) != 0) { - ERROR("Remove file %s failed: %s", dstfile, strerror(errno)); - } + if (ret == 0) { + (void)fflush((FILE *)dstfp); } return ret; diff --git a/src/utils/tar/util_gzip.h b/src/utils/tar/util_gzip.h index 7a96e1189a27891b0f1a8b057f1d683d391acae4..66c07c3b866006d5435a4d60919fccd9fcdfdc62 100644 --- a/src/utils/tar/util_gzip.h +++ b/src/utils/tar/util_gzip.h @@ -23,7 +23,7 @@ extern "C" { int util_gzip_z(const char *srcfile, const char *dstfile, const mode_t mode); // Decompress -int util_gzip_d(const char *srcfile, const char *dstfile, const mode_t mode); +int util_gzip_d(const char *srcfile, const FILE *destfp); #ifdef __cplusplus } diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt index ecf45c28734ced0a985420feb7bf09d9e9a9e5da..8848867e4a32a000f2934658041b0afcca7d4ac1 100644 --- a/test/image/oci/storage/layers/CMakeLists.txt +++ b/test/image/oci/storage/layers/CMakeLists.txt @@ -85,6 +85,7 @@ add_executable(${LAYER_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map/map.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map/rb_tree.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/tar/util_archive.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/tar/util_gzip.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/sha256/sha256.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/err_msg.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/selinux_label.c diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt index 08457a501e2c69de5e89a17240a54d0f9b5aa81c..81fa30f94c8afc04c34aa411901630c5200d2c18 100644 --- a/test/runtime/isula/CMakeLists.txt +++ b/test/runtime/isula/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(${EXE} ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_file.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/util_atomic.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/tar/util_gzip.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c @@ -34,6 +35,7 @@ target_include_directories(${EXE} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/tar ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/services/execution/manager ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/events