diff --git a/gpAux/extensions/gps3ext/include/restful_service.h b/gpAux/extensions/gps3ext/include/restful_service.h index 49ca7c41c686918c65458878599dd8c706b8d213..03c5bca428b10c2e9735bd79e7d26be2b0f258cc 100644 --- a/gpAux/extensions/gps3ext/include/restful_service.h +++ b/gpAux/extensions/gps3ext/include/restful_service.h @@ -19,6 +19,16 @@ enum ResponseStatus { RESPONSE_ERROR, // server error (server return code is not 200) }; +typedef long ResponseCode; + +struct UploadData { + UploadData(const vector& buff) : buffer(buff), currentPosition(0) { + } + + const vector& buffer; + uint64_t currentPosition; +}; + class Response { public: Response() : status(RESPONSE_FAIL) { @@ -82,6 +92,9 @@ class RESTfulService { virtual Response put(const string& url, HTTPHeaders& headers, const map& params, const vector& data) = 0; + + virtual ResponseCode head(const string& url, HTTPHeaders& headers, + const map& params) = 0; }; #endif /* INCLUDE_RESTFUL_SERVICE_H_ */ diff --git a/gpAux/extensions/gps3ext/include/s3interface.h b/gpAux/extensions/gps3ext/include/s3interface.h index 5620be96d45433d772254063edf4337b38546ff0..2270d771c17cb10e8686aefd79b24647a3ef9423 100644 --- a/gpAux/extensions/gps3ext/include/s3interface.h +++ b/gpAux/extensions/gps3ext/include/s3interface.h @@ -76,6 +76,11 @@ class S3Interface { const S3Credential& cred) { throw std::runtime_error("Default implementation must not be called."); } + + virtual bool checkKeyExistence(const string& keyUrl, const string& region, + const S3Credential& cred) { + throw std::runtime_error("Default implementation must not be called."); + } }; class S3Service : public S3Interface { @@ -91,6 +96,8 @@ class S3Service : public S3Interface { S3CompressionType checkCompressionType(const string& keyUrl, const string& region, const S3Credential& cred); + bool checkKeyExistence(const string& keyUrl, const string& region, const S3Credential& cred); + // following two functions are exposed publicly for UT tests void setRESTfulService(RESTfulService* restfullService) { this->restfulService = restfullService; diff --git a/gpAux/extensions/gps3ext/include/s3restful_service.h b/gpAux/extensions/gps3ext/include/s3restful_service.h index 7c9720f5367e42742085d6f3e43f7a30e44dc7f0..92fd139d9c8c84b3e8d5a3b8187d5710014915cd 100644 --- a/gpAux/extensions/gps3ext/include/s3restful_service.h +++ b/gpAux/extensions/gps3ext/include/s3restful_service.h @@ -8,8 +8,11 @@ class S3RESTfulService : public RESTfulService { S3RESTfulService(); virtual ~S3RESTfulService(); Response get(const string& url, HTTPHeaders& headers, const map& params); + Response put(const string& url, HTTPHeaders& headers, const map& params, const vector& data); + + ResponseCode head(const string& url, HTTPHeaders& headers, const map& params); }; #endif /* INCLUDE_S3RESTFUL_SERVICE_H_ */ diff --git a/gpAux/extensions/gps3ext/src/s3interface.cpp b/gpAux/extensions/gps3ext/src/s3interface.cpp index e3fc48cd1f650c97367a198aba69f9dadf91430a..621b2d411399e2b6ee70017544ad8d54177027bb 100644 --- a/gpAux/extensions/gps3ext/src/s3interface.cpp +++ b/gpAux/extensions/gps3ext/src/s3interface.cpp @@ -48,7 +48,7 @@ Response S3Service::getResponseWithRetries(const string &url, HTTPHeaders &heade const map ¶ms, uint64_t retries) { while (retries--) { // declare response here to leverage RVO (Return Value Optimization) - Response response = restfulService->get(url, headers, params); + Response response = this->restfulService->get(url, headers, params); if (response.isSuccess() || (retries == 0)) { return response; }; @@ -387,6 +387,26 @@ S3CompressionType S3Service::checkCompressionType(const string &keyUrl, const st return S3_COMPRESSION_PLAIN; } +bool S3Service::checkKeyExistence(const string &keyUrl, const string ®ion, + const S3Credential &cred) { + HTTPHeaders headers; + map params; + UrlParser parser(keyUrl); + + headers.Add(HOST, parser.getHost()); + headers.Add(X_AMZ_CONTENT_SHA256, "UNSIGNED-PAYLOAD"); + + SignRequestV4("HEAD", &headers, region, parser.getPath(), "", cred); + + ResponseCode code = this->restfulService->head(keyUrl, headers, params); + + if (code == 200 || code == 206) { + return true; + } + + return false; +} + ListBucketResult::~ListBucketResult() { vector::iterator i; for (i = this->contents.begin(); i != this->contents.end(); i++) { diff --git a/gpAux/extensions/gps3ext/src/s3restful_service.cpp b/gpAux/extensions/gps3ext/src/s3restful_service.cpp index 3a093c3b000a0b92b4cae075e55e3edcb28752c7..fb3eaaf0ac70761162e6967c240f83230602a1d4 100644 --- a/gpAux/extensions/gps3ext/src/s3restful_service.cpp +++ b/gpAux/extensions/gps3ext/src/s3restful_service.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "gpcommon.h" #include "s3http_headers.h" @@ -38,15 +39,7 @@ size_t RESTfulServiceWriteFuncCallback(char *ptr, size_t size, size_t nmemb, voi return realsize; } -struct UploadData { - UploadData(const vector &buff) : buffer(buff), currentPosition(0) { - } - - const vector &buffer; - uint64_t currentPosition; -}; - -// curl's read function callback. +// curl's reading function callback. size_t RESTfulServiceReadFuncCallback(char *ptr, size_t size, size_t nmemb, void *userp) { if (QueryCancelPending) { return -1; @@ -56,16 +49,16 @@ size_t RESTfulServiceReadFuncCallback(char *ptr, size_t size, size_t nmemb, void uint64_t dataLeft = data->buffer.size() - data->currentPosition; size_t requestedSize = size * nmemb; - size_t copyItemNum = requestedSize < dataLeft ? nmemb : (dataLeft / size); - size_t copySize = copyItemNum * size; + size_t copiedItemNum = requestedSize < dataLeft ? nmemb : (dataLeft / size); + size_t copiedDataSize = copiedItemNum * size; - if (copySize == 0) return 0; + if (copiedDataSize == 0) return 0; - memcpy(ptr, data->buffer.data() + data->currentPosition, copySize); + memcpy(ptr, data->buffer.data() + data->currentPosition, copiedDataSize); - data->currentPosition += copySize; + data->currentPosition += copiedDataSize; - return copyItemNum; + return copiedItemNum; } // get() will execute HTTP GET RESTful API with given url/headers/params, @@ -146,7 +139,7 @@ Response S3RESTfulService::put(const string &url, HTTPHeaders &headers, headers.CreateList(); - /* configs for download */ + /* options for downloading */ curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); @@ -154,7 +147,7 @@ Response S3RESTfulService::put(const string &url, HTTPHeaders &headers, curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&response); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RESTfulServiceWriteFuncCallback); - /* configs for upload */ + /* options for uploading */ UploadData uploadData(data); curl_easy_setopt(curl, CURLOPT_READDATA, (void *)&uploadData); curl_easy_setopt(curl, CURLOPT_READFUNCTION, RESTfulServiceReadFuncCallback); @@ -208,3 +201,49 @@ Response S3RESTfulService::put(const string &url, HTTPHeaders &headers, return response; } + +// head() will execute HTTP HEAD RESTful API with given url/headers/params, and return the HTTP +// response code. +// +// Currently, this method only return the HTTP code, will be extended if needed in the future +// implementation. +ResponseCode S3RESTfulService::head(const string &url, HTTPHeaders &headers, + const map ¶ms) { + ResponseCode responseCode = -1; + + CURL *curl = curl_easy_init(); + CHECK_OR_DIE_MSG(curl != NULL, "%s", "Failed to create curl handler"); + + headers.CreateList(); + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.GetList()); + + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD"); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + + map::const_iterator iter = params.find("debug"); + if (iter != params.end() && iter->second == "true") { + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + } + + if (s3ext_debug_curl) { + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + } + + CURLcode res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + S3ERROR("curl_easy_perform() failed: %s", curl_easy_strerror(res)); + } else { + // Get the HTTP response status code from HTTP header + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode); + } + + curl_easy_cleanup(curl); + headers.FreeList(); + + return responseCode; +} diff --git a/gpAux/extensions/gps3ext/test/mock_classes.h b/gpAux/extensions/gps3ext/test/mock_classes.h index caba9309c4dd48ff257fd8145b938baf5a8d713e..6ae027bc9bbe8c13a594e7ed11a00032eae00784 100644 --- a/gpAux/extensions/gps3ext/test/mock_classes.h +++ b/gpAux/extensions/gps3ext/test/mock_classes.h @@ -31,6 +31,9 @@ class MockS3RESTfulService : public S3RESTfulService { public: MOCK_METHOD3(get, Response(const string &url, HTTPHeaders &headers, const map ¶ms)); + + MOCK_METHOD3(head, ResponseCode(const string &url, HTTPHeaders &headers, + const map ¶ms)); }; class XMLGenerator { diff --git a/gpAux/extensions/gps3ext/test/s3interface_test.cpp b/gpAux/extensions/gps3ext/test/s3interface_test.cpp index e41715eaeee5b0eeff03ec95c7844796751599db..75ba4c4cebcd0cd18a26d81d5ec46b08a9b7ee5f 100644 --- a/gpAux/extensions/gps3ext/test/s3interface_test.cpp +++ b/gpAux/extensions/gps3ext/test/s3interface_test.cpp @@ -393,3 +393,35 @@ TEST_F(S3ServiceTest, fetchDataWithResponseError) { "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/whatever", region, cred), std::runtime_error); } + +TEST_F(S3ServiceTest, HeadResponse200) { + string url = "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/whatever"; + + EXPECT_CALL(mockRestfulService, head(_, _, _)).WillOnce(Return(200)); + + EXPECT_TRUE(s3service->checkKeyExistence(url, this->region, this->cred)); +} + +TEST_F(S3ServiceTest, HeadResponse206) { + string url = "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/whatever"; + + EXPECT_CALL(mockRestfulService, head(_, _, _)).WillOnce(Return(206)); + + EXPECT_TRUE(s3service->checkKeyExistence(url, this->region, this->cred)); +} + +TEST_F(S3ServiceTest, HeadResponse404) { + string url = "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/whatever"; + + EXPECT_CALL(mockRestfulService, head(_, _, _)).WillOnce(Return(404)); + + EXPECT_FALSE(s3service->checkKeyExistence(url, this->region, this->cred)); +} + +TEST_F(S3ServiceTest, HeadResponse403) { + string url = "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/whatever"; + + EXPECT_CALL(mockRestfulService, head(_, _, _)).WillOnce(Return(403)); + + EXPECT_FALSE(s3service->checkKeyExistence(url, this->region, this->cred)); +} diff --git a/gpAux/extensions/gps3ext/test/s3restful_service_test.cpp b/gpAux/extensions/gps3ext/test/s3restful_service_test.cpp index f8dcf1d5e583d27efd2ccff44a7239783bc04328..f43dfb31eb63dc99f0b3c0f294d8215a8fc3abbe 100644 --- a/gpAux/extensions/gps3ext/test/s3restful_service_test.cpp +++ b/gpAux/extensions/gps3ext/test/s3restful_service_test.cpp @@ -167,3 +167,40 @@ TEST(S3RESTfulService, DISABLED_PutToDummyServer) { EXPECT_EQ(RESPONSE_OK, resp.getStatus()); EXPECT_TRUE(compareVector(data, resp.getRawData())); } + +TEST(S3RESTfulService, HeadWithoutURL) { + HTTPHeaders headers; + map params; + string url; + S3RESTfulService service; + + ResponseCode code = service.head(url, headers, params); + + EXPECT_EQ(-1, code); +} + +TEST(S3RESTfulService, HeadWithWrongURL) { + HTTPHeaders headers; + map params; + string url; + S3RESTfulService service; + + url = "https://www.bing.com/pivotal.html"; + + ResponseCode code = service.head(url, headers, params); + + EXPECT_EQ(404, code); +} + +TEST(S3RESTfulService, HeadWithCorrectURL) { + HTTPHeaders headers; + map params; + string url; + S3RESTfulService service; + + url = "https://www.bing.com/"; + + ResponseCode code = service.head(url, headers, params); + + EXPECT_EQ(200, code); +}