Skip to content

I
iSulad

openeuler / iSulad

通知 15
Star 0
Fork 0
I
iSulad

体验新版 GitCode，发现更多精彩内容 >>

提交 38698319 编写于 作者: O openeuler-ci-bot 提交者： Gitee
浏览文件
操作

!611 concurrent pull layers 

Merge pull request !611 from wangfengtu/fast_pull
上级 b6c5a617 d60fd275
隐藏空白更改
内联 并排
Showing with 339 addition and 188 deletion
src/daemon/common/err_msg.c
浏览文件 @ 38698319
......@@ -89,3 +89,4 @@ void isulad_append_error_message(const char *format, ...)
}
g_isulad_errmsg = result;
}
src/daemon/modules/image/oci/registry/http_request.c
浏览文件 @ 38698319
......@@ -21,6 +21,7 @@
#include <stdlib.h>
#include <strings.h>
#include <time.h>
#include <curl/curl.h>
#include "isula_libutils/log.h"
#include "buffer.h"
......@@ -677,7 +678,18 @@ out:
return ret;
}
int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file)
static int progress(void *p, double dltotal, double dlnow, double ultotal, double ulnow)
{
bool *cancel = p;
if (*cancel) {
// return nonzero code means abort transition
return -1;
}
return 0;
}
int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file,
resp_data_type type)
{
int ret = 0;
struct http_get_options *options = NULL;
......@@ -695,9 +707,15 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo
}
memset(options, 0x00, sizeof(struct http_get_options));
if (type == HEAD_BODY) {
options->with_head = 1;
}
options->with_body = 1;
options->outputtype = HTTP_REQUEST_FILE;
options->output = file;
options->show_progress = 1;
options->progressinfo = &desc->cancel;
options->progress_info_op = progress;
ret = setup_common_options(desc, options, url, custom_headers);
if (ret != 0) {
......@@ -707,7 +725,7 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo
}
ret = http_request(url, options, NULL, 0);
if (ret) {
if (ret != 0) {
ERROR("Failed to get http request: %s", options->errmsg);
isulad_try_set_error_message("%s", options->errmsg);
ret = -1;
......
src/daemon/modules/image/oci/registry/http_request.h
浏览文件 @ 38698319
......@@ -29,7 +29,8 @@ typedef enum {
int http_request_buf(pull_descriptor *desc, const char *url, const char **custom_headers, char **output,
resp_data_type type);
int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file);
int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file,
resp_data_type type);
#ifdef __cplusplus
}
......
src/daemon/modules/image/oci/registry/registry.c
浏览文件 @ 38698319
......@@ -54,6 +54,7 @@
#include "utils_verify.h"
#define MANIFEST_BIG_DATA_KEY "manifest"
#define MAX_CONCURRENT_DOWNLOAD_NUM 5
typedef struct {
pull_descriptor *desc;
......@@ -505,12 +506,6 @@ static int set_cached_info_to_desc(thread_fetch_info *infos, size_t infos_len, p
if (desc->layers[i].diff_id == NULL) {
desc->layers[i].diff_id = util_strdup_s(cache->diffid);
} else if (cache->diffid == NULL) {
ERROR("unexpected NULL diff id");
return -1;
} else if (strcmp(desc->layers[i].diff_id, cache->diffid)) {
ERROR("invalid diff id, expect %s, got %s", desc->layers[i].diff_id, cache->diffid);
return -1;
}
if (desc->layers[i].file == NULL) {
......@@ -1101,27 +1096,50 @@ static void set_cached_layers_info(char *blob_digest, char *diffid, int result,
return;
}
static int fetch_one_layer(thread_fetch_info *info)
static void *fetch_layer_in_thread(void *arg)
{
thread_fetch_info *info = (thread_fetch_info *)arg;
int ret = 0;
char *diffid = NULL;
ret = pthread_detach(pthread_self());
if (ret != 0) {
ERROR("Set thread detach fail");
goto out;
}
prctl(PR_SET_NAME, "fetch_layer");
if (fetch_layer(info->desc, info->index) != 0) {
ERROR("fetch layer %ld failed", info->index);
ret = -1;
goto out;
}
diffid = oci_calc_diffid(info->file);
if (diffid == NULL) {
ERROR("calc diffid for layer %ld failed", info->index);
ret = -1;
goto out;
// calc diffid only if it's schema v1. schema v1 have
// no diff id so we need to calc it. schema v2 have
// diff id in config and we do not want to calc it again
// as it cost too much time.
if (is_manifest_schemav1(info->desc->manifest.media_type)) {
diffid = oci_calc_diffid(info->file);
if (diffid == NULL) {
ERROR("calc diffid for layer %ld failed", info->index);
ret = -1;
goto out;
}
}
out:
// notify to continue downloading
mutex_lock(&g_shared->mutex);
if (ret != 0) {
info->desc->cancel = true;
if (info->desc->errmsg == NULL && g_isulad_errmsg != NULL) {
info->desc->errmsg = util_strdup_s(g_isulad_errmsg);
}
}
DAEMON_CLEAR_ERRMSG();
info->desc->pulling_number--;
set_cached_layers_info(info->blob_digest, diffid, ret, info->file);
if (pthread_cond_broadcast(&g_shared->cond)) {
ERROR("Failed to broadcast");
......@@ -1131,46 +1149,53 @@ out:
free(diffid);
diffid = NULL;
return ret;
return NULL;
}
static int do_fetch(thread_fetch_info *info)
static int add_fetch_task(thread_fetch_info *info)
{
int ret = 0;
int cond_ret = 0;
pthread_t tid = 0;
bool cached_layers_added = true;
cached_layer *cache = NULL;
mutex_lock(&g_shared->mutex);
cache = get_cached_layer(info->blob_digest);
if (cache == NULL) {
// If there are too many download threads, wait until anyone completed.
while (info->desc->pulling_number >= MAX_CONCURRENT_DOWNLOAD_NUM) {
cond_ret = pthread_cond_wait(&g_shared->cond, &g_shared->mutex);
if (cond_ret != 0) {
ERROR("condition wait failed, ret %d", cond_ret);
ret = -1;
goto out;
}
}
}
ret = add_cached_layer(info->blob_digest, info->file);
if (ret != 0) {
ERROR("add fetch info failed, ret %d", cond_ret);
ret = -1;
goto unlock_out;
goto out;
}
cached_layers_added = true;
mutex_unlock(&g_shared->mutex);
if (cache == NULL) {
ret = fetch_one_layer(info);
ret = pthread_create(&tid, NULL, fetch_layer_in_thread, info);
if (ret != 0) {
ERROR("failed to to fetch layer %d", (int)info->index);
ERROR("failed to start thread fetch layer %d", (int)info->index);
goto out;
}
info->desc->pulling_number++;
}
goto out;
unlock_out:
mutex_unlock(&g_shared->mutex);
out:
if (ret != 0 && cached_layers_added) {
mutex_lock(&g_shared->mutex);
del_cached_layer(info->blob_digest, info->file);
mutex_unlock(&g_shared->mutex);
}
mutex_unlock(&g_shared->mutex);
return ret;
}
......@@ -1187,12 +1212,22 @@ static void free_thread_fetch_info(thread_fetch_info *info)
return;
}
static bool all_fetch_complete(thread_fetch_info *infos, size_t infos_len, int *result)
static bool all_fetch_complete(thread_fetch_info *infos, pull_descriptor *desc, int *result)
{
size_t i = 0;
cached_layer *cache = NULL;
size_t infos_len = desc->layers_len;
if (!desc->config.complete) {
return false;
}
*result = 0;
if (desc->config.result != 0) {
*result = desc->config.result;
}
for (i = 0; i < infos_len; i++) {
if (!infos[i].use) {
continue;
......@@ -1216,13 +1251,71 @@ static bool all_fetch_complete(thread_fetch_info *infos, size_t infos_len, int *
return true;
}
static int fetch_layers(pull_descriptor *desc)
static void *fetch_config_in_thread(void *arg)
{
pull_descriptor *desc = (pull_descriptor *)arg;
int ret = 0;
ret = pthread_detach(pthread_self());
if (ret != 0) {
ERROR("Set thread detach fail");
goto out;
}
prctl(PR_SET_NAME, "fetch_config");
ret = fetch_and_parse_config(desc);
if (ret != 0) {
ERROR("fetch and parse config failed for image %s", desc->dest_image_name);
isulad_try_set_error_message("fetch and parse config failed");
goto out;
}
out:
mutex_lock(&g_shared->mutex);
if (ret != 0) {
desc->cancel = true;
if (desc->errmsg == NULL && g_isulad_errmsg != NULL) {
desc->errmsg = util_strdup_s(g_isulad_errmsg);
}
}
DAEMON_CLEAR_ERRMSG();
desc->config.complete = true;
desc->config.result = ret;
if (pthread_cond_broadcast(&g_shared->cond)) {
ERROR("Failed to broadcast");
}
mutex_unlock(&g_shared->mutex);
return NULL;
}
static int add_fetch_config_task(pull_descriptor *desc)
{
pthread_t tid = 0;
// manifest schema1 cann't pull config, the config is composited by
// the history[0].v1Compatibility in manifest and rootfs's diffID
if (is_manifest_schemav1(desc->manifest.media_type)) {
desc->config.complete = true;
desc->config.result = 0;
return 0;
}
if (pthread_create(&tid, NULL, fetch_config_in_thread, desc)) {
ERROR("failed to start thread to fetch config");
return -1;
}
return 0;
}
static int fetch_all(pull_descriptor *desc)
{
size_t i = 0;
size_t j = 0;
int ret = 0;
int sret = 0;
struct layer *l = NULL;
thread_fetch_info *infos = NULL;
char file[PATH_MAX] = { 0 };
int cond_ret = 0;
......@@ -1241,6 +1334,15 @@ static int fetch_layers(pull_descriptor *desc)
return -1;
}
// fetch config in thread
ret = add_fetch_config_task(desc);
if (ret != 0) {
ERROR("add fetch config task failed");
free(infos);
return -1;
}
// fetch layers
for (i = 0; i < desc->layers_len; i++) {
// Skip empty layer
if (desc->layers[i].empty_layer) {
......@@ -1248,37 +1350,27 @@ static int fetch_layers(pull_descriptor *desc)
}
// Skip layer that already exist in local store
if (desc->layers[i].chain_id) {
l = storage_layer_get(without_sha256_prefix(desc->layers[i].chain_id));
if (l != NULL) {
desc->layers[i].already_exist = true;
parent_chain_id = desc->layers[i].chain_id;
free_layer(l);
l = NULL;
continue;
}
}
// Skip layer that already exist in local store for schema1
if (is_manifest_schemav1(desc->manifest.media_type)) {
list = storage_layers_get_by_uncompress_digest(desc->layers[i].digest);
if (list != NULL) {
for (j = 0; j < list->layers_len; j++) {
if ((list->layers[j]->parent == NULL && i == 0) ||
(parent_chain_id != NULL && list->layers[j]->parent != NULL &&
!strcmp(list->layers[j]->parent, without_sha256_prefix(parent_chain_id)))) {
desc->layers[i].already_exist = true;
list = storage_layers_get_by_compress_digest(desc->layers[i].digest);
if (list != NULL) {
for (j = 0; j < list->layers_len; j++) {
if ((list->layers[j]->parent == NULL && i == 0) ||
(parent_chain_id != NULL && list->layers[j]->parent != NULL &&
!strcmp(list->layers[j]->parent, without_sha256_prefix(parent_chain_id)) &&
strcmp(list->layers[j]->uncompressed_digest, list->layers[j]->compressed_digest))) {
desc->layers[i].already_exist = true;
// oci or schema2 get diff id and chain id when get config
if (is_manifest_schemav1(desc->manifest.media_type)) {
desc->layers[i].diff_id = util_strdup_s(list->layers[j]->uncompressed_digest);
desc->layers[i].chain_id = util_string_append(list->layers[j]->id, SHA256_PREFIX);
parent_chain_id = desc->layers[i].chain_id;
break;
}
parent_chain_id = desc->layers[i].chain_id;
break;
}
free_layer_list(list);
list = NULL;
if (desc->layers[i].already_exist) {
continue;
}
}
free_layer_list(list);
list = NULL;
if (desc->layers[i].already_exist) {
continue;
}
}
......@@ -1290,7 +1382,7 @@ static int fetch_layers(pull_descriptor *desc)
if (sret < 0 || (size_t)sret >= sizeof(file)) {
ERROR("Failed to sprintf file for layer %d", (int)i);
ret = -1;
goto out;
break;
}
infos[i].desc = desc;
......@@ -1299,14 +1391,19 @@ static int fetch_layers(pull_descriptor *desc)
infos[i].file = util_strdup_s(file);
infos[i].blob_digest = util_strdup_s(desc->layers[i].digest);
ret = do_fetch(&infos[i]);
ret = add_fetch_task(&infos[i]);
if (ret != 0) {
goto out;
infos[i].use = false;
break;
}
}
if (ret != 0) {
desc->cancel = true;
}
// wait until all pulled or cancelled
mutex_lock(&g_shared->mutex);
while (!all_fetch_complete(infos, desc->layers_len, &result)) {
while (!all_fetch_complete(infos, desc, &result)) {
cond_ret = pthread_cond_wait(&g_shared->cond, &g_shared->mutex);
if (cond_ret != 0) {
ERROR("condition wait for all layers to complete failed, ret %d", cond_ret);
......@@ -1323,13 +1420,13 @@ static int fetch_layers(pull_descriptor *desc)
if (set_cached_info_to_desc(infos, desc->layers_len, desc) != 0) {
ERROR("set cached infos to desc failed");
}
} else if (desc->errmsg != NULL) {
ERROR("pull image %s failed: %s", desc->dest_image_name, desc->errmsg);
isulad_try_set_error_message(desc->errmsg);
}
mutex_unlock(&g_shared->mutex);
out:
free_layer_list(list);
list = NULL;
for (i = 0; i < desc->layers_len; i++) {
if (ret != 0 && infos[i].use) {
mutex_lock(&g_shared->mutex);
......@@ -1484,18 +1581,7 @@ static int registry_fetch(pull_descriptor *desc, bool *reuse)
goto out;
}
// manifest schema1 cann't pull config, the config is composited by
// the history[0].v1Compatibility in manifest and rootfs's diffID
if (!is_manifest_schemav1(desc->manifest.media_type)) {
ret = fetch_and_parse_config(desc);
if (ret != 0) {
ERROR("fetch and parse config failed for image %s", desc->dest_image_name);
isulad_try_set_error_message("fetch and parse config failed");
goto out;
}
}
ret = fetch_layers(desc);
ret = fetch_all(desc);
if (ret != 0) {
ERROR("fetch layers failed for image %s", desc->dest_image_name);
isulad_try_set_error_message("fetch layers failed");
......@@ -1594,6 +1680,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio
desc->use_decrypted_key = conf_get_use_decrypted_key_flag();
desc->skip_tls_verify = options->skip_tls_verify;
desc->insecure_registry = options->insecure_registry;
desc->cancel = false;
if (options->auth.username != NULL && options->auth.password != NULL) {
desc->username = util_strdup_s(options->auth.username);
......
src/daemon/modules/image/oci/registry/registry_apiv2.c
浏览文件 @ 38698319
......@@ -43,6 +43,8 @@
#define DOCKER_API_VERSION_HEADER "Docker-Distribution-Api-Version: registry/2.0"
#define MAX_ACCEPT_LEN 128
// retry 5 times
#define RETRY_TIMES 5
static int parse_http_header(char *resp_buf, size_t buf_size, struct parsed_http_message *message)
{
......@@ -446,7 +448,7 @@ static int registry_request(pull_descriptor *desc, char *path, char **custom_hea
}
DEBUG("resp=%s", *output_buffer);
} else {
ret = http_request_file(desc, url, (const char **)headers, file);
ret = http_request_file(desc, url, (const char **)headers, file, type);
if (ret != 0) {
ERROR("http request file failed, url: %s", url);
goto out;
......@@ -478,7 +480,7 @@ static int check_content_type(const char *content_type)
return -1;
}
static int parse_manifests_info(char *http_head, char **content_type, char **digest)
static int parse_manifest_head(char *http_head, char **content_type, char **digest)
{
int ret = 0;
struct parsed_http_message *message = NULL;
......@@ -589,7 +591,56 @@ out:
return ret;
}
static int fetch_manifests_info(pull_descriptor *desc, char **content_type, char **digest)
static int split_head_body(char *file, char **http_head)
{
int ret = 0;
char *all = NULL;
char *head = NULL;
char *deli = "\r\n\r\n"; // default delimiter, may change in later process
char *body = NULL;
all = util_read_text_file(file);
if (all == NULL) {
ERROR("read file %s failed", file);
return -1;
}
head = strstr(all, "HTTP/1.1");
if (head == NULL) {
ERROR("No HTTP/1.1 found");
ret = -1;
goto out;
}
body = strstr(head, deli);
if (body == NULL) {
deli = "\n\n";
body = strstr(head, deli);
if (body == NULL) {
ERROR("No body found, data=%s", head);
ret = -1;
goto out;
}
}
body += strlen(deli);
ret = util_write_file(file, body, strlen(body), 0600);
if (ret != 0) {
ERROR("rewrite body to file failed");
ret = -1;
goto out;
}
*body = 0;
*http_head = util_strdup_s(head);
out:
free(all);
all = NULL;
return ret;
}
static int fetch_manifest_list(pull_descriptor *desc, char *file, char **content_type, char **digest)
{
int ret = 0;
int sret = 0;
......@@ -615,13 +666,19 @@ static int fetch_manifests_info(pull_descriptor *desc, char **content_type, char
goto out;
}
ret = registry_request(desc, path, custom_headers, NULL, &http_head, HEAD_ONLY);
ret = registry_request(desc, path, custom_headers, file, NULL, HEAD_BODY);
if (ret != 0) {
ERROR("registry: Get %s failed", path);
goto out;
}
ret = parse_manifests_info(http_head, content_type, digest);
ret = split_head_body(file, &http_head);
if (ret != 0) {
ERROR("registry: Split %s to head body failed", file);
goto out;
}
ret = parse_manifest_head(http_head, content_type, digest);
if (ret != 0) {
ret = -1;
goto out;
......@@ -643,6 +700,7 @@ static int fetch_data(pull_descriptor *desc, char *path, char *file, char *conte
int sret = 0;
char accept[MAX_ELEMENT_SIZE] = { 0 };
char **custom_headers = NULL;
int retry_times = RETRY_TIMES;
// digest can be NULL
if (desc == NULL || path == NULL || file == NULL || content_type == NULL) {
......@@ -663,19 +721,31 @@ static int fetch_data(pull_descriptor *desc, char *path, char *file, char *conte
goto out;
}
ret = registry_request(desc, path, custom_headers, file, NULL, BODY_ONLY);
if (ret != 0) {
ERROR("registry: Get %s failed", path);
goto out;
}
// If content is signatured, digest is for payload but not fetched data
if (strcmp(content_type, DOCKER_MANIFEST_SCHEMA1_PRETTYJWS) && digest != NULL) {
if (!sha256_valid_digest_file(file, digest)) {
ERROR("data from %s does not have digest %s", path, digest);
ret = -1;
while (retry_times > 0) {
retry_times--;
ret = registry_request(desc, path, custom_headers, file, NULL, BODY_ONLY);
if (ret != 0) {
if (retry_times > 0) {
continue;
}
ERROR("registry: Get %s failed", path);
desc->cancel = true;
goto out;
}
// If content is signatured, digest is for payload but not fetched data
if (strcmp(content_type, DOCKER_MANIFEST_SCHEMA1_PRETTYJWS) && digest != NULL) {
if (!sha256_valid_digest_file(file, digest)) {
if (retry_times > 0) {
continue;
}
ret = -1;
ERROR("data from %s does not have digest %s", path, digest);
desc->cancel = true;
goto out;
}
}
break;
}
out:
......@@ -869,7 +939,7 @@ out:
return ret;
}
static int fetch_manifests_data(pull_descriptor *desc, char *file, char **content_type, char **digest)
static int fetch_manifest_data(pull_descriptor *desc, char *file, char **content_type, char **digest)
{
int ret = 0;
int sret = 0;
......@@ -881,23 +951,6 @@ static int fetch_manifests_data(pull_descriptor *desc, char *file, char **conten
return -1;
}
if (*digest != NULL) {
sret = snprintf(path, sizeof(path), "/v2/%s/manifests/%s", desc->name, *digest);
} else {
sret = snprintf(path, sizeof(path), "/v2/%s/manifests/%s", desc->name, desc->tag);
}
if (sret < 0 || (size_t)sret >= sizeof(path)) {
ERROR("Failed to sprintf path for manifest");
ret = -1;
goto out;
}
ret = fetch_data(desc, path, file, *content_type, *digest);
if (ret != 0) {
ERROR("registry: Get %s failed", path);
goto out;
}
// If it's manifest list, we must choose the manifest which match machine's architecture to download.
if (!strcmp(*content_type, DOCKER_MANIFEST_SCHEMA2_LIST) || !strcmp(*content_type, OCI_INDEX_V1_JSON)) {
ret = select_manifest(file, content_type, digest);
......@@ -947,13 +1000,13 @@ int fetch_manifest(pull_descriptor *desc)
return -1;
}
ret = fetch_manifests_info(desc, &content_type, &digest);
ret = fetch_manifest_list(desc, file, &content_type, &digest);
if (ret != 0) {
ret = -1;
goto out;
}
ret = fetch_manifests_data(desc, file, &content_type, &digest);
ret = fetch_manifest_data(desc, file, &content_type, &digest);
if (ret != 0) {
ret = -1;
goto out;
......
src/daemon/modules/image/oci/registry_type.c
浏览文件 @ 38698319
......@@ -94,6 +94,9 @@ void free_pull_desc(pull_descriptor *desc)
free(desc->certs_dir);
desc->certs_dir = NULL;
free(desc->errmsg);
desc->errmsg = NULL;
free(desc->blobpath);
desc->blobpath = NULL;
free(desc->protocol);
......
src/daemon/modules/image/oci/registry_type.h
浏览文件 @ 38698319
......@@ -53,6 +53,8 @@ typedef struct {
// Downloaded file path
char *file;
types_timestamp_t create_time;
bool complete;
int result;
} config_blob;
typedef struct {
......@@ -89,6 +91,10 @@ typedef struct {
char *key_file;
char *certs_dir;
int pulling_number;
bool cancel;
char *errmsg;
char *blobpath;
char *protocol;
bool skip_tls_verify;
......
src/daemon/modules/image/oci/storage/image_store/image_store.c
浏览文件 @ 38698319
......@@ -731,7 +731,7 @@ static int get_layers_from_manifest(const registry_manifest_schema1 *manifest, l
layers[index].media_type = util_strdup_s(DOCKER_IMAGE_LAYER_TAR_GZIP);
layers[index].digest = util_strdup_s(manifest->fs_layers[i]->blob_sum);
list = storage_layers_get_by_uncompress_digest(layers[index].digest);
list = storage_layers_get_by_compress_digest(layers[index].digest);
if (list != NULL) {
for (j = 0; j < list->layers_len; j++) {
if ((list->layers[j]->parent == NULL && index == 0) ||
......
src/daemon/modules/image/oci/storage/storage.c
浏览文件 @ 38698319
......@@ -239,7 +239,7 @@ out:
return ret;
}
struct layer_list *storage_layers_get_by_uncompress_digest(const char *digest)
struct layer_list *storage_layers_get_by_compress_digest(const char *digest)
{
int ret = 0;
struct layer_list *layers = NULL;
......
src/daemon/modules/image/oci/storage/storage.h
浏览文件 @ 38698319
......@@ -150,7 +150,7 @@ char *storage_img_get_image_id(const char *img_name);
/* layer operations */
int storage_layer_create(const char *layer_id, storage_layer_create_opts_t *opts);
struct layer_list *storage_layers_get_by_uncompress_digest(const char *digest);
struct layer_list *storage_layers_get_by_compress_digest(const char *digest);
struct layer *storage_layer_get(const char *layer_id);
......
src/utils/http/http.c
浏览文件 @ 38698319
......@@ -266,7 +266,10 @@ static void check_buf_len(struct http_get_options *options, char *errbuf, CURLco
return;
}
}
ERROR("curl response error code %d, error message: %s", curl_result, errbuf);
free(options->errmsg);
options->errmsg = util_strdup_s(errbuf);
options->errcode = curl_result;
return;
}
......@@ -346,8 +349,11 @@ int http_request(const char *url, struct http_get_options *options, long *respon
/* set URL to get here */
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);
/* complete connection within 15 seconds */
/* complete connection within 30 seconds */
curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, 30L);
/* if less than 1k data is received in 30s, abort */
curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, 30L);
/* provide a buffer to store errors in */
curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, errbuf);
curl_easy_setopt(curl_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
......
src/utils/http/http.h
浏览文件 @ 38698319
......@@ -70,6 +70,7 @@ struct http_get_options {
char *key_file;
char *errmsg;
int errcode;
void *progressinfo;
progress_info_func progress_info_op;
......
test/image/oci/registry/data/oci/index
浏览文件 @ 38698319
HTTP/1.1 200 OK
Server: nginx/1.10.1
Date: Thu, 02 Jul 2020 09:14:16 GMT
Content-Type: application/vnd.oci.image.index.v1+json
Content-Length: 2080
Connection: keep-alive
Docker-Content-Digest: sha256:bd28e852703450d93220e6733a9f0901b92cd558911528b03fdba56156ae0a02
Docker-Distribution-Api-Version: registry/2.0
Etag: "sha256:bd28e852703450d93220e6733a9f0901b92cd558911528b03fdba56156ae0a02"
{
"manifests": [
{
......
test/image/oci/registry/data/oci/manifest_head 已删除 100644 → 0
浏览文件 @ b6c5a617
HTTP/1.1 200 OK
Server: nginx/1.10.1
Date: Thu, 02 Jul 2020 09:14:16 GMT
Content-Type: application/vnd.oci.image.index.v1+json
Content-Length: 2080
Connection: keep-alive
Docker-Content-Digest: sha256:bd28e852703450d93220e6733a9f0901b92cd558911528b03fdba56156ae0a02
Docker-Distribution-Api-Version: registry/2.0
Etag: "sha256:bd28e852703450d93220e6733a9f0901b92cd558911528b03fdba56156ae0a02"
test/image/oci/registry/data/v1/manifest_body → test/image/oci/registry/data/v1/manifest
浏览文件 @ 38698319
HTTP/1.1 200 OK
Server: nginx/1.12.1
Date: Tue, 30 Jun 2020 09:33:47 GMT
Content-Type: application/vnd.docker.distribution.manifest.v1+prettyjws
Content-Length: 9124
Connection: close
Docker-Content-Digest: sha256:3080be9aab91b28e49af8551b70dccf54c11e5a9c21719ab4d058dbddb65bc96
X-Frame-Options: DENY
Strict-Transport-Security: max-age=63072000; preload
{
"schemaVersion": 1,
"name": "coreos/etcd",
......
test/image/oci/registry/data/v1/manifest_head 已删除 100644 → 0
浏览文件 @ b6c5a617
HTTP/1.1 200 OK
Server: nginx/1.12.1
Date: Tue, 30 Jun 2020 09:33:47 GMT
Content-Type: application/vnd.docker.distribution.manifest.v1+prettyjws
Content-Length: 9124
Connection: close
Docker-Content-Digest: sha256:3080be9aab91b28e49af8551b70dccf54c11e5a9c21719ab4d058dbddb65bc96
X-Frame-Options: DENY
Strict-Transport-Security: max-age=63072000; preload
test/image/oci/registry/data/v2/manifest_head 已删除 100644 → 0
浏览文件 @ b6c5a617
HTTP/1.1 200 OK
Server: nginx/1.10.1
Date: Thu, 02 Jul 2020 09:14:16 GMT
Content-Type: application/vnd.docker.distribution.manifest.list.v2+json
Content-Length: 2080
Connection: keep-alive
Docker-Content-Digest: sha256:9ddee63a712cea977267342e8750ecbc60d3aab25f04ceacfa795e6fce341793
Docker-Distribution-Api-Version: registry/2.0
Etag: "sha256:9ddee63a712cea977267342e8750ecbc60d3aab25f04ceacfa795e6fce341793"
test/image/oci/registry/data/v2/manifest_list
浏览文件 @ 38698319
HTTP/1.1 200 OK
Server: nginx/1.10.1
Date: Thu, 02 Jul 2020 09:14:16 GMT
Content-Type: application/vnd.docker.distribution.manifest.list.v2+json
Content-Length: 2080
Connection: keep-alive
Docker-Content-Digest: sha256:9ddee63a712cea977267342e8750ecbc60d3aab25f04ceacfa795e6fce341793
Docker-Distribution-Api-Version: registry/2.0
Etag: "sha256:9ddee63a712cea977267342e8750ecbc60d3aab25f04ceacfa795e6fce341793"
{"manifests":[{"digest":"sha256:2131f09e4044327fd101ca1fd4043e6f3ad921ae7ee901e9142e6e36b354a907","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"amd64","os":"linux"},"size":527},{"digest":"sha256:ea84577ce8331aaceefd586104ba283201b89b5a614b10ec44b9884722db49d8","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v5"},"size":527},{"digest":"sha256:296361e74fe78e932cdd807743b5e37469518194f95c042135a6c3320ca52ef1","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v6"},"size":527},{"digest":"sha256:5cbe4404234f93a5401b58e0c50408d5c9caace822b70867e4f3e787be83eee9","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v7"},"size":527},{"digest":"sha256:134252904112f8563a17a360957d9ad192e5c1e77463e04be74e71cffd4b41ba","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm64","os":"linux","variant":"v8"},"size":527},{"digest":"sha256:414aeb860595d7078cbe87abaeed05157d6b44907fbd7db30e1cfba9b6902448","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"386","os":"linux"},"size":527},{"digest":"sha256:116dccaef9ca8b121565a39bd568ede437f084c94bb0642d2aba6b441e38d2f8","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"mips64le","os":"linux"},"size":527},{"digest":"sha256:5477c332ec926f8221e82a6c9e37dd9d84a401e3b5f71ba7d498956552c880ac","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"ppc64le","os":"linux"},"size":528},{"digest":"sha256:c304d497f3e0f87f8457401787df738f6f6e62b367bfd7c5f73f5b880b30ab4f","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"s390x","os":"linux"},"size":528}],"mediaType":"application\/vnd.docker.distribution.manifest.list.v2+json","schemaVersion":2}
\ No newline at end of file
test/image/oci/registry/registry_ut.cpp
浏览文件 @ 38698319
......@@ -88,30 +88,6 @@ protected:
NiceMock<MockStorage> m_storage_mock;
};
int invokeHttpRequestBuf(pull_descriptor *desc, const char *url, const char **custom_headers, char **output,
resp_data_type type)
{
std::string file;
std::string data_path = get_dir() + "/data/";
if (!strcmp(url, "https://quay.io/v2/")) {
file = data_path + "v1_ping_head";
} else if (!strcmp(url, "https://quay.io/v2/coreos/etcd/manifests/v3.3.17-arm64")) {
file = data_path + "v1_manifest_head";
} else {
ERROR("%s not match failed", url);
return -1;
}
*output = util_read_text_file(file.c_str());
if (*output == NULL) {
ERROR("read file %s failed", file.c_str());
return -1;
}
return 0;
}
int invokeHttpRequestV1(const char *url, struct http_get_options *options, long *response_code, int recursive_len)
{
std::string file;
......@@ -129,7 +105,7 @@ int invokeHttpRequestV1(const char *url, struct http_get_options *options, long
file = data_path + "ping_head";
}
} else if (!strcmp(url, "https://quay.io/v2/coreos/etcd/manifests/v3.3.17-arm64")) {
file = data_path + "manifest_head";
file = data_path + "manifest";
} else if (util_has_prefix(url, "https://auth.quay.io")) {
token_count++;
if (token_count == 2) {
......@@ -137,8 +113,6 @@ int invokeHttpRequestV1(const char *url, struct http_get_options *options, long
} else {
file = data_path + "token_body";
}
} else if (util_has_prefix(url, "https://quay.io/v2/coreos/etcd/manifests/sha256")) {
file = data_path + "manifest_body";
} else if (util_has_prefix(url, "https://quay.io/v2/coreos/etcd/blobs/sha256")) {
file = std::string("");
} else {
......@@ -176,6 +150,7 @@ int invokeHttpRequestV2(const char *url, struct http_get_options *options, long
char *data = NULL;
int64_t size = 0;
Buffer *output_buffer = (Buffer *)options->output;
static bool retry = true;
// Test insecure registry, assume registry cann't support https.
if (util_has_prefix(url, "https://")) {
......@@ -186,14 +161,16 @@ int invokeHttpRequestV2(const char *url, struct http_get_options *options, long
if (!strcmp(url, "http://hub-mirror.c.163.com/v2/")) {
file = data_path + "ping_head";
} else if (!strcmp(url, "http://hub-mirror.c.163.com/v2/library/busybox/manifests/latest")) {
file = data_path + "manifest_head";
} else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/manifests/sha256:9ddee63a")) {
file = data_path + "manifest_list";
} else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/manifests/sha256:2131f09e")) {
file = data_path + "manifest_body";
} else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:c7c37e47")) {
file = data_path + "config";
} else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:91f30d77")) {
if (retry) {
retry = false;
return -1;
}
file = data_path + "0";
} else {
ERROR("%s not match failed", url);
......@@ -238,8 +215,6 @@ int invokeHttpRequestOCI(const char *url, struct http_get_options *options, long
if (!strcmp(url, "https://hub-mirror.c.163.com/v2/")) {
file = data_path + "ping_head";
} else if (!strcmp(url, "https://hub-mirror.c.163.com/v2/library/busybox/manifests/latest")) {
file = data_path + "manifest_head";
} else if (util_has_prefix(url, "https://hub-mirror.c.163.com/v2/library/busybox/manifests/sha256:bd28e852")) {
file = data_path + "index";
} else if (util_has_prefix(url, "https://hub-mirror.c.163.com/v2/library/busybox/manifests/sha256:106429d7")) {
file = data_path + "manifest_body";
......@@ -365,12 +340,12 @@ struct layer * invokeStorageLayerGet(const char *layer_id)
return NULL;
}
struct layer_list *invokeStorageLayersGetByUncompressDigest(const char *digest)
struct layer_list *invokeStorageLayersGetByCompressDigest(const char *digest)
{
int ret = 0;
struct layer_list *list = NULL;
list = (struct layer_list*)util_common_calloc_s(sizeof(struct layer_list*));
list = (struct layer_list*)util_common_calloc_s(sizeof(struct layer_list));
if (list == NULL) {
ERROR("out of memory");
return NULL;
......@@ -655,8 +630,8 @@ TEST_F(RegistryUnitTest, test_pull_already_exist)
.WillRepeatedly(Invoke(invokeHttpRequestV1));
EXPECT_CALL(m_storage_mock, StorageLayerGet(::testing::_))
.WillRepeatedly(Invoke(invokeStorageLayerGet));
EXPECT_CALL(m_storage_mock, StorageLayersGetByUncompressDigest(::testing::_))
.WillRepeatedly(Invoke(invokeStorageLayersGetByUncompressDigest));
EXPECT_CALL(m_storage_mock, StorageLayersGetByCompressDigest(::testing::_))
.WillRepeatedly(Invoke(invokeStorageLayersGetByCompressDigest));
ASSERT_NE(registry_pull(&options), 0);
}
......
test/image/oci/storage/images/storage_images_ut.cpp
浏览文件 @ 38698319
......@@ -180,7 +180,7 @@ struct layer_list *new_layer_list(const char *parent, const char *id, const char
return list;
}
struct layer_list *invokeStorageLayersGetByUncompressDigest(const char *digest)
struct layer_list *invokeStorageLayersGetByCompressDigest(const char *digest)
{
if (strcmp(digest, "sha256:270b8855c17de6980aff6cb060c7d95c35e018cfb30c85e5a0b7810ad5620761") == 0) {
return new_layer_list(nullptr, "777a4eeaaf3deda1634b11faa1f3b205899bfeaef1b35793011b0cc498f2f569",
......@@ -263,8 +263,8 @@ TEST_F(StorageImagesCompatibilityUnitTest, test_load_v1_image)
std::string dir = GetDirectory() + "/data";
ASSERT_STRNE(cleanpath(dir.c_str(), store_real_path, sizeof(store_real_path)), nullptr);
EXPECT_CALL(m_storage_mock, StorageLayersGetByUncompressDigest(_))
.WillRepeatedly(Invoke(invokeStorageLayersGetByUncompressDigest));
EXPECT_CALL(m_storage_mock, StorageLayersGetByCompressDigest(_))
.WillRepeatedly(Invoke(invokeStorageLayersGetByCompressDigest));
EXPECT_CALL(m_storage_mock, FreeLayerList(_))
.WillRepeatedly(Invoke(invokeFreeLayerList));
opts.storage_root = strdup(store_real_path);
......
test/image/oci/storage/layers/storage_layers_ut.cpp
浏览文件 @ 38698319
......@@ -306,4 +306,4 @@ TEST_F(StorageLayersUnitTest, test_layer_store_by_uncompress_digest)
ASSERT_EQ(layers[0]->uncompress_size, 1672256);
free_layer_list(layer_list);
}
\ No newline at end of file
}
test/mocks/storage_mock.cc
浏览文件 @ 38698319
......@@ -24,10 +24,10 @@ void MockStorage_SetMock(MockStorage *mock)
g_storage_mock = mock;
}
struct layer_list *storage_layers_get_by_uncompress_digest(const char *digest)
struct layer_list *storage_layers_get_by_compress_digest(const char *digest)
{
if (g_storage_mock != NULL) {
return g_storage_mock->StorageLayersGetByUncompressDigest(digest);
return g_storage_mock->StorageLayersGetByCompressDigest(digest);
}
return NULL;
......
test/mocks/storage_mock.h
浏览文件 @ 38698319
......@@ -22,7 +22,7 @@
class MockStorage {
public:
virtual ~MockStorage() = default;
MOCK_METHOD1(StorageLayersGetByUncompressDigest, struct layer_list * (const char *digest));
MOCK_METHOD1(StorageLayersGetByCompressDigest, struct layer_list * (const char *digest));
MOCK_METHOD1(FreeLayerList, void(struct layer_list *ptr));
MOCK_METHOD4(StorageImgCreate, int(const char *id, const char *parent_id, const char *metadata,
struct storage_img_create_options *opts));
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或