s3downloader_test.cpp

#include "s3downloader.cpp"
#include "gtest/gtest.h"

volatile bool QueryCancelPending = false;

TEST(OffsetMgr, simple) {
    OffsetMgr *o = new OffsetMgr(4096, 1000);
    Range r = o->NextOffset();
    EXPECT_EQ(r.offset, 0);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 1000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 2000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 3000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 4000);
    EXPECT_EQ(r.len, 96);
    delete o;
}

TEST(OffsetMgr, reset) {
    OffsetMgr *o = new OffsetMgr(1024, 100);

    o->NextOffset();
    o->NextOffset();
    o->Reset(333);
    Range r = o->NextOffset();

    EXPECT_EQ(r.offset, 333);
    EXPECT_EQ(r.len, 100);
    delete o;
}

#ifdef FAKE_TEST
#define HOSTSTR "localhost"
#define BUCKETSTR "metro.pivotal.io"
TEST(ListBucket, fake) {
    ListBucketResult *r = ListBucket_FakeHTTP(HOSTSTR, BUCKETSTR);

    ASSERT_NE(r, (void *)NULL);
    EXPECT_EQ(16, r->contents.size());

    delete r;
}
#endif  // FAKE_TEST

#define S3REGION "us-west-2"
#define S3BUCKET "s3test.pivotal.io"
#define S3PREFIX "dataset1/small17"

#ifdef AWS_TEST

TEST(ListBucket, s3) {
    InitConfig("test/s3.conf", "default");

    S3Credential g_cred = {s3ext_accessid, s3ext_secret};

    ListBucketResult *r =
        ListBucket("https", S3REGION, S3BUCKET, S3PREFIX, g_cred);

    ASSERT_NE(r, (void *)NULL);
    EXPECT_EQ(16, r->contents.size());

    // leave this snippet here, for future debugging
    /*
     * char urlbuf[256];
     * vector<BucketContent *>::iterator i;
     * for (i = r->contents.begin(); i != r->contents.end(); i++) {
     *     BucketContent *p = *i;
     *     sprintf(urlbuf, "https://s3-%s.amazonaws.com/%s/%s", S3REGION,
     * S3BUCKET,
     *             p->Key().c_str());
     *     printf("%s, %d\n", urlbuf, p->Size());
     * }
     */

    delete r;
}

TEST(ListBucket, s3_1024) {
    InitConfig("test/s3.conf", "default");

    S3Credential g_cred = {s3ext_accessid, s3ext_secret};

    ListBucketResult *r =
        ListBucket("https", S3REGION, S3BUCKET, "1024files", g_cred);

    ASSERT_NE(r, (void *)NULL);
    EXPECT_EQ(1024, r->contents.size());

    delete r;
}

TEST(ListBucket, s3_5120) {
    InitConfig("test/s3.conf", "default");

    S3Credential g_cred = {s3ext_accessid, s3ext_secret};

    ListBucketResult *r =
        ListBucket("https", S3REGION, S3BUCKET, "5120files", g_cred);

    ASSERT_NE(r, (void *)NULL);
    EXPECT_EQ(5120, r->contents.size());

    delete r;
}

#endif  // AWS_TEST

void DownloadTest(const char *url, const char *region, uint64_t file_size,
                  const char *md5_str, uint8_t thread_num, uint64_t chunk_size,
                  uint64_t buffer_size, bool use_credential) {
    InitConfig("test/s3.conf", "default");

    S3Credential g_cred = {s3ext_accessid, s3ext_secret};

    uint64_t buf_len = buffer_size;
    char *buf = (char *)malloc(buffer_size);
    Downloader *d = new Downloader(thread_num);
    MD5Calc m;
    bool result = false;

    ASSERT_NE((void *)NULL, buf);

    if (use_credential) {
        result = d->init(url, region, file_size, chunk_size, &g_cred);
    } else {
        result = d->init(url, "", file_size, chunk_size, NULL);
    }

    ASSERT_TRUE(result);

    while (1) {
        ASSERT_TRUE(d->get(buf, buf_len));
        if (buf_len == 0) {
            break;
        }
        m.Update(buf, buf_len);
        buf_len = buffer_size;
    }

    ASSERT_STREQ(md5_str, m.Get());

    d->destroy();
    delete d;
    free(buf);
}

#ifdef FAKE_TEST
void HTTPDownloaderTest(const char *url, uint64_t file_size,
                        const char *md5_str, uint8_t thread_num,
                        uint64_t chunk_size, uint64_t buffer_size) {
    return DownloadTest(url, NULL, file_size, md5_str, thread_num, chunk_size,
                        buffer_size, false);
}

TEST(HTTPDownloader, divisible) {
    HTTPDownloaderTest("http://localhost/testdata/8M", 8388608,
                       "22129b81bf8f96a06a8b7f3d2a683588", 4, 4 * 1024,
                       16 * 1024);
}

TEST(HTTPDownloader, equal) {
    HTTPDownloaderTest("http://localhost/testdata/8M", 8388608,
                       "22129b81bf8f96a06a8b7f3d2a683588", 4, 16 * 1024,
                       16 * 1024);
}

TEST(HTTPDownloader, one_byte) {
    HTTPDownloaderTest("http://localhost/testdata/8K", 8192,
                       "f92ad4ae5f16a8d04f87e801af5115dc", 4, 4 * 1024, 1);
}

TEST(HTTPDownloader, over_flow) {
    HTTPDownloaderTest("http://localhost/testdata/8M", 8388608,
                       "22129b81bf8f96a06a8b7f3d2a683588", 4, 7 * 1024,
                       15 * 1024);
}

TEST(HTTPDownloader, multi_thread) {
    HTTPDownloaderTest("http://localhost/testdata/1M", 1048576,
                       "06427456b575a3880936b4ae43448082", 64, 4 * 1024,
                       511 * 1023);
}

TEST(HTTPDownloader, single_thread) {
    HTTPDownloaderTest("http://localhost/testdata/1M", 1048576,
                       "06427456b575a3880936b4ae43448082", 1, 4 * 1024,
                       511 * 1023);
}

TEST(HTTPDownloader, random_parameters_1M) {
    HTTPDownloaderTest("http://localhost/testdata/1M", 1048576,
                       "06427456b575a3880936b4ae43448082", 3, 3 * 29,
                       571 * 1023);
}

#ifdef BIG_FILE_TEST
TEST(HTTPDownloader, random_parameters_1G) {
    HTTPDownloaderTest("http://localhost/testdata/1G", 1073741824,
                       "a2cb2399eb8bc97084aed673e5d09f4d", 9, 42 * 1023,
                       513 * 1025 * 37);
}
#endif

TEST(HTTPDownloader, random_parameters_8M) {
    HTTPDownloaderTest("http://localhost/testdata/8M", 8388608,
                       "22129b81bf8f96a06a8b7f3d2a683588", 77, 7 * 10211,
                       777 * 1025);
}

TEST(HTTPDownloader, random_parameters_64M) {
    HTTPDownloaderTest("http://localhost/testdata/64M", 67108864,
                       "0871a7464a7ff85564f4f95b9ac77321", 51, 7 * 1053,
                       77 * 87);
}

TEST(HTTPDownloader, random_parameters_256M) {
    HTTPDownloaderTest("http://localhost/testdata/256M", 268435456,
                       "9cb7c287fdd5a44378798d3e75d2d2a6", 3, 1 * 10523,
                       77 * 879);
}
#endif  // FAKE_TEST

#ifdef AWS_TEST
void S3DwonloadTest(const char *url, const char *region, uint64_t file_size,
                    const char *md5_str, uint8_t thread_num,
                    uint64_t chunk_size, uint64_t buffer_size) {
    InitConfig("test/s3.conf", "default");

    return DownloadTest(url, region, file_size, md5_str, thread_num, chunk_size,
                        buffer_size, true);
}

TEST(S3Downloader, simple) {
    printf("Try downloading data0014\n");
    S3DwonloadTest(
        "http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/"
        "data0014",
        "us-west-2", 4420346, "68c4a63b721e7af0ae945ce109ca87ad", 4,
        1024 * 1024, 65536);

    printf("Try downloading data0016\n");
    S3DwonloadTest(
        "http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/"
        "data0016",
        "us-west-2", 2536018, "0fd502a303eb8f138f5916ec357721b1", 4,
        1024 * 1024, 65536);
}

TEST(S3Downloader, httpssimple) {
    printf("Try downloading data0014\n");
    S3DwonloadTest(
        "http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/"
        "data0014",
        "us-west-2", 4420346, "68c4a63b721e7af0ae945ce109ca87ad", 4,
        1024 * 1024, 65536);

    printf("Try downloading data0016\n");
    S3DwonloadTest(
        "http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/"
        "data0016",
        "us-west-2", 2536018, "0fd502a303eb8f138f5916ec357721b1", 4,
        1024 * 1024, 65536);
}

#ifdef BIG_FILE_TEST
TEST(S3Downloader, bigfile) {
    // InitLog();
    // s3ext_loglevel = EXT_DEBUG;
    // s3ext_logtype = STDERR_LOG;
    printf("Try downloading big file 1\n");
    S3DwonloadTest(
        "http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/"
        "airlinedata1.csv",
        "us-west-2", 4664425994, "f5811ad92c994f1d6913d5338575fe38", 4,
        64 * 1024 * 1024, 65536);
}
#endif

#endif  // AWS_TEST