s3ext: source cleanup and reorganization

remove scripts directory.
remove LICENSE file since it's the same as GPDB's.
remove dead codes and meaningless comments.
reorder header includings alphabetically.
add a missing unit test for find_replace().
rename testing sql files.
rename source files.
optimize compiling flags.

gpAux/extensions/gps3ext/.gitignore

@@ -48,3 +48,5 @@ core
 *.gcda
 *.gcno
 s3test
+
+s3.conf

gpAux/extensions/gps3ext/LICENSE

-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-

gpAux/extensions/gps3ext/Makefile

-MODULE_big = gps3ext
-OBJS = lib/http_parser.o lib/ini.o src/gps3conf.o src/gps3ext.o src/S3Common.o src/S3ExtWrapper.o src/S3Downloader.o src/utils.o src/S3Log.o
+# Options
+DEBUG_S3 = n
+DEBUG_S3_CURL = n
 
-PG_CPPFLAGS = -std=c++98 -fPIC -I$(libpq_srcdir) -Isrc -Ilib -I/usr/include/libxml2 -I$(libpq_srcdir)/postgresql/server/utils -g
+# Flags
 SHLIB_LINK = -lstdc++ -lxml2 -lpthread -lcrypto -lcurl
+PG_CPPFLAGS = -O2 -g -std=c++98 -fPIC -I$(libpq_srcdir) -Isrc -Ilib -I/usr/include/libxml2 -I$(libpq_srcdir)/postgresql/server/utils
+
+ifeq ($(DEBUG_S3),y)
+	PG_CPPFLAGS += -DDEBUG_S3
+endif
+
+ifeq ($(DEBUG_S3_CURL),y)
+	PG_CPPFLAGS += -DDEBUG_S3_CURL
+endif
+
+# Targets
+MODULE_big = gps3ext
+OBJS = lib/http_parser.o lib/ini.o src/gps3ext.o src/s3conf.o src/s3common.o src/s3wrapper.o src/s3downloader.o src/s3utils.o src/s3log.o
 
+# Launch
 PGXS := $(shell pg_config --pgxs)
 include $(PGXS)

gpAux/extensions/gps3ext/Makefile.test

+# Options
+DEBUG_S3 = y
+DEBUG_S3_CURL = n
+AWS_TEST = y
+FAKE_TEST = y
+BIG_FILE_TEST = y
+
+# Flags
 CPP = g++
-CFLAGS = -g3 -std=c++98 -fPIC -fprofile-arcs -ftest-coverage -DDEBUGS3 -DAWSTEST -DFAKETEST
 INCLUDES = -Isrc -Ilib -I/usr/include/libxml2
 LDFLAGS = -lpthread -lcrypto -lcurl -lxml2 -lgcov
+CFLAGS = -O2 -g3 -std=c++98 -fPIC -fprofile-arcs -ftest-coverage
 
-testcources = src/gps3conf_test.cpp src/utils_test.cpp src/S3Downloader_test.cpp src/S3Common_test.cpp src/S3ExtWrapper_test.cpp src/S3Log_test.cpp
+ifeq ($(DEBUG_S3),y)
+	CFLAGS += -DDEBUG_S3
+endif
+
+ifeq ($(DEBUG_S3_CURL),y)
+	CFLAGS += -DDEBUG_S3_CURL
+endif
+
+ifeq ($(AWS_TEST),y)
+	CFLAGS += -DAWS_TEST
+endif
+
+ifeq ($(FAKE_TEST),y)
+	CFLAGS += -DFAKE_TEST
+endif
+
+ifeq ($(BIG_FILE_TEST),y)
+	CFLAGS += -DBIG_FILE_TEST
+endif
+
+# Google TEST
+testcources = src/s3conf_test.cpp src/s3utils_test.cpp src/s3downloader_test.cpp src/s3common_test.cpp src/s3wrapper_test.cpp src/s3log_test.cpp
 
 testapp = s3test
 testobjs = $(testcources:.cpp=.o)
@@ -44,8 +73,8 @@ coverage: test
 	@gcov -r $(testcources) | grep -v test.cpp | grep -A 2 "src/.*.cpp"
 
 clean:
-	rm -f $(testapp) src/*.o src/*.a lib/*.o *.o *.a
 	rm -f *.gcov src/*.gcov src/*.gcda src/*.gcno
+	rm -f $(testapp) *.so src/*.o src/*.a lib/*.o *.o *.a
 
 distclean: clean
 	rm -f tags cscope.*

gpAux/extensions/gps3ext/scripts/decsg

-#!/usr/bin/python
-
-import sys
-
-if __name__ == "__main__":
-    print ''.join(chr(int(i,16)) for i in sys.argv[1:])

gpAux/extensions/gps3ext/scripts/gendoc

-#!/bin/bash
-
-gtags
-htags --suggest -t 'Welcome to GPDB S3 source code!' doc
-

gpAux/extensions/gps3ext/scripts/prepare.sh

-#!/bin/bash
-
-# S3Common.h S3Downloader.h utils.h utils.cpp S3Downloader.cpp S3Common.cpp extlib/http_parser.cpp include/http_parser.h .
-ln -sf library/S3Common.h S3Common.h
-ln -sf library/S3Downloader.h S3Downloader.h
-ln -sf library/utils.h utils.h
-ln -sf library/utils.cpp utils.cpp
-ln -sf library/S3Downloader.cpp S3Downloader.cpp
-ln -sf library/S3Common.cpp S3Common.cpp
-ln -sf library/extlib/http_parser.cpp http_parser.cpp
-ln -sf library/include/http_parser.h http_parser.h
-
-

gpAux/extensions/gps3ext/scripts/std.cfg

-<?xml version="1.0"?>
-<def>
-  <memory>
-    <dealloc>free</dealloc>
-
-    <alloc init="false">malloc</alloc>
-    <alloc init="true">calloc</alloc>
-    <alloc init="true">strdup</alloc>
-    <alloc init="true">strndup</alloc>
-  </memory>
-
-  <resource>
-    <dealloc>fclose</dealloc>
-    <alloc init="true">fopen</alloc>
-  </resource>
-
-  <function name="fopen">   <noreturn>false</noreturn> </function>
-  <function name="fclose">  <noreturn>false</noreturn> </function>
-  <function name="fflush">  <noreturn>false</noreturn> <leak-ignore/> </function>
-
-  <function name="isalha">  <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="isalnum"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="iscntrl"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="isdigit"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="isxdigit"> <noreturn>false</noreturn> <leak-ignore/> <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="isgraph"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="islower"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="isupper"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="isprint"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="ispunct"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-  <function name="isspace"> <noreturn>false</noreturn> <leak-ignore/>  <arg nr="1"><valid>0-255</valid></arg> </function>
-
-  <function name="memchr">  <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-  <function name="memcmp">  <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-  <function name="memcpy">  <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-  <function name="memmove"> <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-  <function name="memset">  <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-
-  <function name="strcat">  <noreturn>false</noreturn> <leak-ignore/> </function>
-  <function name="strchr">  <noreturn>false</noreturn> <leak-ignore/> </function>
-  <function name="strcpy">  <noreturn>false</noreturn> <leak-ignore/> </function>
-  <function name="strcmp">  <noreturn>false</noreturn> <leak-ignore/> </function>
-  <function name="strdup">  <noreturn>false</noreturn> <leak-ignore/> </function>
-  <function name="strlen">  <noreturn>false</noreturn> <leak-ignore/> </function>
-  <function name="strncat"> <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-  <function name="strncpy"> <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-  <function name="strncmp"> <noreturn>false</noreturn> <leak-ignore/> <arg nr="3"><not-bool/><valid>0-</valid></arg> </function>
-  <function name="strstr">  <noreturn>false</noreturn> <leak-ignore/> </function>
-
-  <function name="strtol">   <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-  <function name="strtoll">  <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-  <function name="strtoul">  <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-  <function name="strtoull"> <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-  <function name="wcstol">   <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-  <function name="wcstoll">  <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-  <function name="wcstoul">  <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-  <function name="wcstoull"> <leak-ignore/> <arg nr="3"><valid>0,2-36</valid></arg> </function>
-</def>
-

gpAux/extensions/gps3ext/src/gps3ext.cpp

@@ -2,27 +2,28 @@
 
 #include <cstdlib>
 
+#include <pthread.h>
+#include <signal.h>
+
 #include "postgres.h"
+
 #include "funcapi.h"
 
 #include "access/extprotocol.h"
 #include "catalog/pg_proc.h"
+#include "fmgr.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
 #include "utils/memutils.h"
-#include "fmgr.h"
 
-#include "S3ExtWrapper.h"
-#include "S3Common.h"
-#include "S3Log.h"
-#include "utils.h"
+#include <openssl/err.h>
 
 #include "gps3ext.h"
-#include "gps3conf.h"
-
-#include <signal.h>
-#include <pthread.h>
-#include <openssl/err.h>
+#include "s3common.h"
+#include "s3conf.h"
+#include "s3log.h"
+#include "s3utils.h"
+#include "s3wrapper.h"
 
 /* Do the module magic dance */
 
@@ -164,14 +165,6 @@ Datum s3_import(PG_FUNCTION_ARGS) {
                                       "configurations and net connection",
                                       s3ext_segid, s3ext_segnum)));
         }
-        /*
-                  if(strcasecmp(parsed_url->protocol, p_name) != 0) {
-                  elog(ERROR, "internal error: s3prot called with a different
-                  protocol
-                  (%s)",
-                  parsed_url->protocol);
-                  }
-        */
 
         EXTPROTOCOL_SET_USER_CTX(fcinfo, myData);
 

gpAux/extensions/gps3ext/src/gps3ext.h

 #ifndef _GPS3EXT_H_
 #define _GPS3EXT_H_
 #include <string>
+
 #include <signal.h>
+#include <sys/types.h>
 
 // GPDB's global val
 extern volatile bool QueryCancelPending;

gpAux/extensions/gps3ext/src/S3Common.cpp → gpAux/extensions/gps3ext/src/s3common.cpp

-#include "S3Common.h"
-#include "utils.h"
-
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
@@ -11,110 +8,13 @@
 #include <sstream>
 #include <string>
 
-#include "gps3conf.h"
+#include "s3common.h"
+#include "s3conf.h"
+#include "s3utils.h"
 
 using std::string;
 using std::stringstream;
 
-bool SignGETv2(HeaderContent *h, string path_with_query,
-               const S3Credential &cred) {
-    char timestr[65];
-    unsigned char tmpbuf[20];  // SHA_DIGEST_LENGTH is 20
-
-    // CONTENT_LENGTH is not a part of StringToSign
-    h->Add(CONTENTLENGTH, "0");
-
-    gethttpnow(timestr);
-    h->Add(DATE, timestr);
-    stringstream sstr;
-    sstr << "GET\n\n\n" << timestr << "\n" << path_with_query;
-    if (!sha1hmac(sstr.str().c_str(), tmpbuf, cred.secret.c_str(),
-                  strlen(cred.secret.c_str()))) {
-        return false;
-    }
-    // S3DEBUG("%s", sstr.str().c_str());
-    char *signature =
-        Base64Encode((char *)tmpbuf, 20);  // SHA_DIGEST_LENGTH is 20
-    // S3DEBUG("%s", signature);
-    sstr.clear();
-    sstr.str("");
-    sstr << "AWS " << cred.keyid << ":" << signature;
-    free(signature);
-    // S3DEBUG("%s", sstr.str().c_str());
-    h->Add(AUTHORIZATION, sstr.str());
-
-    return true;
-}
-
-bool SignPUTv2(HeaderContent *h, string path_with_query,
-               const S3Credential &cred) {
-    char timestr[65];
-    unsigned char tmpbuf[20];  // SHA_DIGEST_LENGTH is 20
-    string typestr;
-
-    gethttpnow(timestr);
-    h->Add(DATE, timestr);
-    stringstream sstr;
-
-    typestr = h->Get(CONTENTTYPE);
-
-    sstr << "PUT\n\n" << typestr << "\n" << timestr << "\n" << path_with_query;
-    if (!sha1hmac(sstr.str().c_str(), tmpbuf, cred.secret.c_str(),
-                  strlen(cred.secret.c_str()))) {
-        return false;
-    }
-    char *signature =
-        Base64Encode((char *)tmpbuf, 20);  // SHA_DIGEST_LENGTH is 20
-    sstr.clear();
-    sstr.str("");
-    sstr << "AWS " << cred.keyid << ":" << signature;
-    free(signature);
-    h->Add(AUTHORIZATION, sstr.str());
-
-    return true;
-}
-
-bool SignPOSTv2(HeaderContent *h, string path_with_query,
-                const S3Credential &cred) {
-    char timestr[65];
-    unsigned char tmpbuf[20];  // SHA_DIGEST_LENGTH is 20
-    // string md5str;
-
-    gethttpnow(timestr);
-    h->Add(DATE, timestr);
-    stringstream sstr;
-    // md5str = h->Get(CONTENTMD5);
-    const char *typestr = h->Get(CONTENTTYPE);
-
-    if (typestr != NULL) {
-        sstr << "POST\n"
-             << "\n"
-             << typestr << "\n"
-             << timestr << "\n"
-             << path_with_query;
-    } else {
-        sstr << "POST\n"
-             << "\n"
-             << "\n"
-             << timestr << "\n"
-             << path_with_query;
-    }
-    // printf("%s\n", sstr.str().c_str());
-    if (!sha1hmac(sstr.str().c_str(), tmpbuf, cred.secret.c_str(),
-                  strlen(cred.secret.c_str()))) {
-        return false;
-    }
-    char *signature =
-        Base64Encode((char *)tmpbuf, 20);  // SHA_DIGEST_LENGTH is 20
-    sstr.clear();
-    sstr.str("");
-    sstr << "AWS " << cred.keyid << ":" << signature;
-    free(signature);
-    h->Add(AUTHORIZATION, sstr.str());
-
-    return true;
-}
-
 bool SignRequestV4(string method, HeaderContent *h, string region, string path,
                    string query, const S3Credential &cred) {
     time_t t;
@@ -332,15 +232,16 @@ uint64_t ParserCallback(void *contents, uint64_t size, uint64_t nmemb,
 
 // invoked by s3_import(), need to be exception safe
 char *get_opt_s3(const char *url, const char *key) {
-    const char *key_f = NULL;
+    const char *key_start = NULL;
     const char *key_tailing = NULL;
-    char *key_val = NULL;
-    int val_len = 0;
+    char *value = NULL;
+    int value_len = 0;
 
     if (!url || !key) {
         return NULL;
     }
 
+    // construct the key to search " key="
     char *key2search = (char *)malloc(strlen(key) + 3);
     if (!key2search) {
         S3ERROR("Can't allocate memory for string");
@@ -354,43 +255,46 @@ char *get_opt_s3(const char *url, const char *key) {
     key2search[key_len + 1] = '=';
     key2search[key_len + 2] = 0;
 
-    // printf("key2search=%s\n", key2search);
+    // get the whole options string (strip "url ")
     const char *delimiter = " ";
     const char *options = strstr(url, delimiter);
-    if (!options) {  // no options string
+    if (!options) {
         goto FAIL;
     }
-    key_f = strstr(options, key2search);
-    if (key_f == NULL) {
+
+    // get the string " key=blah1 key2=blah2 ..."
+    key_start = strstr(options, key2search);
+    if (key_start == NULL) {
         goto FAIL;
     }
 
-    key_f += strlen(key2search);
-    if (*key_f == ' ') {
+    // get the string "blah1 key2=blah2 ..."
+    key_start += strlen(key2search);
+    if (*key_start == ' ') {
         goto FAIL;
     }
-    // printf("key_f=%s\n", key_f);
 
-    key_tailing = strstr(key_f, delimiter);
-    // printf("key_tailing=%s\n", key_tailing);
-    val_len = 0;
+    // get the string "key2=blah2 ..."
+    key_tailing = strstr(key_start, delimiter);
+    // get the length of "blah1"
     if (key_tailing) {
-        val_len = strlen(key_f) - strlen(key_tailing);
+        value_len = strlen(key_start) - strlen(key_tailing);
     } else {
-        val_len = strlen(key_f);
+        value_len = strlen(key_start);
     }
 
-    key_val = (char *)malloc(val_len + 1);
-    if (!key_val) {
+    // get the string "blah1"
+    value = (char *)malloc(value_len + 1);
+    if (!value) {
         goto FAIL;
     }
 
-    memcpy(key_val, key_f, val_len);
-    key_val[val_len] = 0;
+    memcpy(value, key_start, value_len);
+    value[value_len] = 0;
 
     free(key2search);
 
-    return key_val;
+    return value;
 FAIL:
     if (key2search) {
         free(key2search);

gpAux/extensions/gps3ext/src/S3Common.h → gpAux/extensions/gps3ext/src/s3common.h

 #ifndef __S3_COMMON_H__
 #define __S3_COMMON_H__
 
-#include <curl/curl.h>
 #include <map>
 #include <string>
-#include "http_parser.h"
-using std::string;
 
+#include <curl/curl.h>
 #include <libxml/parser.h>
 #include <libxml/tree.h>
 
-#include "S3Log.h"
+#include "http_parser.h"
+#include "s3log.h"
+using std::string;
 
 struct S3Credential {
     string keyid;
@@ -45,15 +45,6 @@ class HeaderContent {
     std::map<HeaderField, string> fields;
 };
 
-bool SignGETv2(HeaderContent* h, string path_with_query,
-               const S3Credential& cred);
-
-bool SignPUTv2(HeaderContent* h, string path_with_query,
-               const S3Credential& cred);
-
-bool SignPOSTv2(HeaderContent* h, string path_with_query,
-                const S3Credential& cred);
-
 bool SignRequestV4(string method, HeaderContent* h, string region, string path,
                    string query, const S3Credential& cred);
 

gpAux/extensions/gps3ext/src/S3Common_test.cpp → gpAux/extensions/gps3ext/src/s3common_test.cpp

-#include "S3Common.cpp"
+#include "s3common.cpp"
 #include "gtest/gtest.h"
 #include "http_parser.cpp"
 
-TEST(S3Common, UrlParser) {
+TEST(Common, UrlParser) {
     UrlParser *p = new UrlParser(
         "https://www.google.com/search?sclient=psy-ab&site=&source=hp");
     EXPECT_NE((void *)NULL, p);
@@ -15,7 +15,7 @@ TEST(S3Common, UrlParser) {
     }
 }
 
-TEST(S3Common, UrlParser_LongURL) {
+TEST(Common, UrlParser_LongURL) {
     UrlParser *p = new UrlParser(
         "http://s3-us-west-2.amazonaws.com/metro.pivotal.io/test/"
         "data1234?partNumber=1&uploadId=."
@@ -35,26 +35,22 @@ TEST(S3Common, UrlParser_LongURL) {
 #define RANGESTR "1-10000"
 #define MD5STR "xxxxxxxxxxxxxxxxxxx"
 
-TEST(S3Common, HeaderContent) {
+TEST(Common, HeaderContent) {
     HeaderContent *h = new HeaderContent();
     EXPECT_NE((void *)NULL, h);
 
-    // test Add
     if (h) {
         ASSERT_TRUE(h->Add(HOST, HOSTSTR));
         ASSERT_TRUE(h->Add(RANGE, RANGESTR));
         ASSERT_TRUE(h->Add(CONTENTMD5, MD5STR));
     }
 
-// test Get
-#if 0
     if (h) {
         EXPECT_STREQ(HOSTSTR, h->Get(HOST));
         EXPECT_STREQ(RANGESTR, h->Get(RANGE));
         EXPECT_STREQ(MD5STR, h->Get(CONTENTMD5));
     }
-#endif
-    // test GetList
+
     if (h) {
         curl_slist *l = h->GetList();
         ASSERT_NE((void *)NULL, l);
@@ -62,13 +58,15 @@ TEST(S3Common, HeaderContent) {
             curl_slist_free_all(l);
         }
     }
+
     if (h) delete h;
 }
 
-TEST(S3Common, UrlOptions) {
+TEST(Common, UrlOptions) {
     char *option = NULL;
-    EXPECT_STREQ("secret_test",
-                 option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test",
+    EXPECT_STREQ(
+        "secret_test",
+        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test",
                             "secret"));
     free(option);
 
@@ -79,60 +77,66 @@ TEST(S3Common, UrlOptions) {
             "accessid"));
     free(option);
 
-    EXPECT_STREQ("3456789",
-                 option = get_opt_s3("s3://neverland.amazonaws.com chunksize=3456789",
+    EXPECT_STREQ(
+        "3456789",
+        option = get_opt_s3("s3://neverland.amazonaws.com chunksize=3456789",
                             "chunksize"));
     free(option);
 
-    EXPECT_STREQ(
-        "secret_test",
-        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
-                   "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
-                   "secret"));
+    EXPECT_STREQ("secret_test",
+                 option = get_opt_s3(
+                     "s3://neverland.amazonaws.com secret=secret_test "
+                     "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
+                     "secret"));
     free(option);
 
-    EXPECT_STREQ(
-        "\".\\!@#$%^&*()DFGHJK\"",
-        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
-                   "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
-                   "accessid"));
+    EXPECT_STREQ("\".\\!@#$%^&*()DFGHJK\"",
+                 option = get_opt_s3(
+                     "s3://neverland.amazonaws.com secret=secret_test "
+                     "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
+                     "accessid"));
     free(option);
 
-    EXPECT_STREQ(
-        "3456789",
-        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
-                   "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
-                   "chunksize"));
+    EXPECT_STREQ("3456789",
+                 option = get_opt_s3(
+                     "s3://neverland.amazonaws.com secret=secret_test "
+                     "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789",
+                     "chunksize"));
     free(option);
 
-    EXPECT_STREQ("secret_test",
-                 option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
+    EXPECT_STREQ(
+        "secret_test",
+        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
                             "blah=whatever accessid=\".\\!@#$%^&*()DFGHJK\" "
                             "chunksize=3456789 KingOfTheWorld=sanpang",
                             "secret"));
     free(option);
 
-    EXPECT_STREQ("secret_test",
-                 option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
+    EXPECT_STREQ(
+        "secret_test",
+        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
                             "blah= accessid=\".\\!@#$%^&*()DFGHJK\" "
                             "chunksize=3456789 KingOfTheWorld=sanpang",
                             "secret"));
     free(option);
 
-    EXPECT_STREQ("3456789",
-                 option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
+    EXPECT_STREQ(
+        "3456789",
+        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
                             "chunksize=3456789 KingOfTheWorld=sanpang ",
                             "chunksize"));
     free(option);
 
-    EXPECT_STREQ("3456789",
-                 option = get_opt_s3("s3://neverland.amazonaws.com   secret=secret_test "
+    EXPECT_STREQ(
+        "3456789",
+        option = get_opt_s3("s3://neverland.amazonaws.com   secret=secret_test "
                             "chunksize=3456789  KingOfTheWorld=sanpang ",
                             "chunksize"));
     free(option);
 
-    EXPECT_STREQ("=sanpang",
-                 option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
+    EXPECT_STREQ(
+        "=sanpang",
+        option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
                             "chunksize=3456789 KingOfTheWorld==sanpang ",
                             "KingOfTheWorld"));
     free(option);
@@ -144,36 +148,38 @@ TEST(S3Common, UrlOptions) {
     EXPECT_EQ((char *)NULL,
               option = get_opt_s3("s3://neverland.amazonaws.com", "secret"));
 
-    EXPECT_EQ((char *)NULL,
-              option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
-                         "blah=whatever accessid= chunksize=3456789 "
-                         "KingOfTheWorld=sanpang",
-                         "accessid"));
-
     EXPECT_EQ(
         (char *)NULL,
         option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
-                   "blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
-                   ""));
+                            "blah=whatever accessid= chunksize=3456789 "
+                            "KingOfTheWorld=sanpang",
+                            "accessid"));
+
+    EXPECT_EQ((char *)NULL,
+              option = get_opt_s3(
+                  "s3://neverland.amazonaws.com secret=secret_test "
+                  "blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
+                  ""));
+
+    EXPECT_EQ((char *)NULL,
+              option = get_opt_s3(
+                  "s3://neverland.amazonaws.com secret=secret_test "
+                  "blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
+                  NULL));
 
     EXPECT_EQ(
         (char *)NULL,
         option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
-                   "blah=whatever chunksize=3456789 KingOfTheWorld=sanpang",
-                   NULL));
-
-    EXPECT_EQ((char *)NULL,
-              option = get_opt_s3("s3://neverland.amazonaws.com secret=secret_test "
-                         "chunksize=3456789 KingOfTheWorld=sanpang ",
-                         "chunk size"));
+                            "chunksize=3456789 KingOfTheWorld=sanpang ",
+                            "chunk size"));
 }
 
-TEST(S3Common, TruncateOptions) {
+TEST(Common, TruncateOptions) {
     char *truncated = NULL;
 
-    EXPECT_STREQ(
-        "s3://neverland.amazonaws.com",
-        truncated = truncate_options("s3://neverland.amazonaws.com secret=secret_test"));
+    EXPECT_STREQ("s3://neverland.amazonaws.com",
+                 truncated = truncate_options(
+                     "s3://neverland.amazonaws.com secret=secret_test"));
     free(truncated);
 
     EXPECT_STREQ(
@@ -182,16 +188,16 @@ TEST(S3Common, TruncateOptions) {
             "s3://neverland.amazonaws.com accessid=\".\\!@#$%^&*()DFGHJK\""));
     free(truncated);
 
-    EXPECT_STREQ(
-        "s3://neverland.amazonaws.com",
-        truncated = truncate_options("s3://neverland.amazonaws.com secret=secret_test "
-                         "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789"));
+    EXPECT_STREQ("s3://neverland.amazonaws.com",
+                 truncated = truncate_options(
+                     "s3://neverland.amazonaws.com secret=secret_test "
+                     "accessid=\".\\!@#$%^&*()DFGHJK\" chunksize=3456789"));
     free(truncated);
 
-    EXPECT_STREQ(
-        "s3://neverland.amazonaws.com",
-        truncated = truncate_options("s3://neverland.amazonaws.com secret=secret_test "
-                         "blah= accessid=\".\\!@#$%^&*()DFGHJK\" "
-                         "chunksize=3456789 KingOfTheWorld=sanpang"));
+    EXPECT_STREQ("s3://neverland.amazonaws.com",
+                 truncated = truncate_options(
+                     "s3://neverland.amazonaws.com secret=secret_test "
+                     "blah= accessid=\".\\!@#$%^&*()DFGHJK\" "
+                     "chunksize=3456789 KingOfTheWorld=sanpang"));
     free(truncated);
 }

gpAux/extensions/gps3ext/src/gps3conf.cpp → gpAux/extensions/gps3ext/src/s3conf.cpp

-#include <cstring>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/types.h>
 #include <arpa/inet.h>
-#include <string>
 #include <errno.h>
+#include <fcntl.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
 #include <unistd.h>
-#include <cstdio>
 #include <cstdarg>
-#include <sys/stat.h>
-#include <fcntl.h>
+#include <cstdio>
+#include <cstring>
 #include <sstream>
+#include <string>
 
-#include "S3Log.h"
-#include "utils.h"
-#include "gps3conf.h"
 #include "gps3ext.h"
+#include "s3conf.h"
+#include "s3log.h"
+#include "s3utils.h"
 
 //#include <cdb/cdbvars.h>
 
-#ifndef DEBUGS3
+#ifndef DEBUG_S3
 extern "C" {
 void write_log(const char* fmt, ...) __attribute__((format(printf, 1, 2)));
 }
@@ -62,7 +62,7 @@ Config* s3cfg = NULL;
 bool InitConfig(string conf_path, string section /*not used currently*/) {
     try {
         if (conf_path == "") {
-#ifndef DEBUGS3
+#ifndef DEBUG_S3
             write_log("Config file is not specified\n");
 #endif
             return false;
@@ -72,7 +72,7 @@ bool InitConfig(string conf_path, string section /*not used currently*/) {
 
         s3cfg = new Config(conf_path);
         if (!s3cfg || !s3cfg->Handle()) {
-#ifndef DEBUGS3
+#ifndef DEBUG_S3
             write_log("Failed to parse config file\n");
 #endif
             if (s3cfg) {
@@ -95,7 +95,7 @@ bool InitConfig(string conf_path, string section /*not used currently*/) {
         s3ext_secret = cfg->Get("default", "secret", "");
         s3ext_token = cfg->Get("default", "token", "");
 
-#ifdef DEBUGS3
+#ifdef DEBUG_S3
 // s3ext_loglevel = EXT_DEBUG;
 // s3ext_logtype = LOCAL_LOG;
 #endif
@@ -156,7 +156,7 @@ bool InitConfig(string conf_path, string section /*not used currently*/) {
         content = cfg->Get("default", "encryption", "true");
         s3ext_encryption = to_bool(content);
 
-#ifdef DEBUGS3
+#ifdef DEBUG_S3
         s3ext_segid = 0;
         s3ext_segnum = 1;
 #else

gpAux/extensions/gps3ext/src/gps3conf.h → gpAux/extensions/gps3ext/src/s3conf.h

@@ -57,8 +57,7 @@ extern struct sockaddr_un s3ext_logserverpath;
 extern int s3ext_low_speed_limit;
 extern int s3ext_low_speed_time;
 
-// not thread safe!!
-// Called only once.
+// not thread safe!! call it only once.
 bool InitConfig(string path, string section);
 
 void ClearConfig();

gpAux/extensions/gps3ext/src/gps3conf_test.cpp → gpAux/extensions/gps3ext/src/s3conf_test.cpp

+#include "s3conf.cpp"
 #include "gtest/gtest.h"
 #include "ini.cpp"
-#include "gps3conf.cpp"
-#include <cstdlib>
 
 TEST(Config, basic) {
     InitConfig("test/s3test.conf", "");
@@ -10,7 +9,7 @@ TEST(Config, basic) {
     EXPECT_STREQ("accessid_test", s3ext_accessid.c_str());
     EXPECT_STREQ("ABCDEFGabcdefg", s3ext_token.c_str());
 
-#ifdef DEBUGS3
+#ifdef DEBUG_S3
     EXPECT_EQ(0, s3ext_segid);
     EXPECT_EQ(1, s3ext_segnum);
 #endif

gpAux/extensions/gps3ext/src/S3Downloader.cpp → gpAux/extensions/gps3ext/src/s3downloader.cpp

-#include "S3Downloader.h"
-
-#include <algorithm>  // std::min
+#include <unistd.h>
+#include <algorithm>
 #include <iostream>
 #include <sstream>
 
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
+
+#include <curl/curl.h>
 #include <libxml/parser.h>
 #include <libxml/tree.h>
 
-#include <curl/curl.h>
-#include "S3Log.h"
 #include "gps3ext.h"
-#include "utils.h"
-#define __STDC_FORMAT_MACROS
-#include <inttypes.h>
+#include "s3downloader.h"
+#include "s3log.h"
+#include "s3utils.h"
 
-#include <unistd.h>
 using std::stringstream;
 
 OffsetMgr::OffsetMgr(uint64_t m, uint64_t c)
@@ -159,9 +159,7 @@ uint64_t BlockingBuffer::Fill() {
         }
     }
     this->status = BlockingBuffer::STATUS_READY;
-    // if (this->realsize >= 0) {
     pthread_cond_signal(&this->stat_cond);
-    //}
 
     pthread_mutex_unlock(&this->stat_mutex);
     return (readlen == -1) ? -1 : this->realsize;
@@ -289,7 +287,6 @@ RETRY:
 }
 
 void Downloader::destroy() {
-    // if error
     for (int i = 0; i < this->num; i++) {
         if (this->threads && this->threads[i]) pthread_cancel(this->threads[i]);
     }
@@ -326,7 +323,9 @@ HTTPFetcher::HTTPFetcher(string url, OffsetMgr *o)
     : BlockingBuffer(url, o), urlparser(url.c_str()), method(GET) {
     this->curl = curl_easy_init();
     if (this->curl) {
-        // curl_easy_setopt(this->curl, CURLOPT_VERBOSE, 1L);
+#if DEBUG_S3_CURL
+        curl_easy_setopt(this->curl, CURLOPT_VERBOSE, 1L);
+#endif
         // curl_easy_setopt(curl, CURLOPT_PROXY, "127.0.0.1:8080");
         curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
         curl_easy_setopt(this->curl, CURLOPT_WRITEFUNCTION, WriterCallback);
@@ -348,14 +347,13 @@ bool HTTPFetcher::SetMethod(Method m) {
 
 bool HTTPFetcher::AddHeaderField(HeaderField f, string v) {
     if (v == "") {
-        // log warning
         S3INFO("Skip adding empty field for %s", GetFieldString(f));
         return false;
     }
     return this->headers.Add(f, v);
 }
 
-// buffer size should be at lease len
+// buffer size should be at least len
 // read len data from offest
 // returning -1 mearns error, pls don't set chunksize to uint64_t(-1) for now
 uint64_t HTTPFetcher::fetchdata(uint64_t offset, char *data, uint64_t len) {
@@ -487,7 +485,9 @@ xmlParserCtxtPtr DoGetXML(string region, string bucket, string url,
 
     if (curl) {
         curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-        // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#if DEBUG_S3_CURL
+        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#endif
         curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L);
     } else {
         S3ERROR("Can't create curl instance, no enough memory?");
@@ -530,7 +530,7 @@ xmlParserCtxtPtr DoGetXML(string region, string bucket, string url,
 
     if (res != CURLE_OK) {
         S3ERROR("curl_easy_perform() failed: %s", curl_easy_strerror(res));
-        if (xml.ctxt) {  // cleanup
+        if (xml.ctxt) {
             xmlDocPtr doc = xml.ctxt->myDoc;
             xmlFreeParserCtxt(xml.ctxt);
             xmlFreeDoc(doc);
@@ -740,13 +740,14 @@ ListBucketResult::~ListBucketResult() {
 ListBucketResult *ListBucket_FakeHTTP(string host, string bucket) {
     std::stringstream sstr;
     sstr << "http://" << host << "/" << bucket;
-    // sstr << host << "/" << bucket;
 
     CURL *curl = curl_easy_init();
 
     if (curl) {
         curl_easy_setopt(curl, CURLOPT_URL, sstr.str().c_str());
-        // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#if DEBUG_S3_CURL
+        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#endif
         // curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L);
     } else {
         return NULL;
@@ -777,9 +778,10 @@ ListBucketResult *ListBucket_FakeHTTP(string host, string bucket) {
         S3ERROR("Failed to create xml node");
         return NULL;
     }
+
     ListBucketResult *result = new ListBucketResult();
+
     if (!result) {
-        // allocate fail
         xmlFreeParserCtxt(xml.ctxt);
         xmlFreeDoc(doc);
         return NULL;

gpAux/extensions/gps3ext/src/S3Downloader.h → gpAux/extensions/gps3ext/src/s3downloader.h

@@ -7,21 +7,20 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <cstddef>
+#include <cstdlib>
 #include <iostream>
 #include <map>
 #include <string>
 #include <vector>
-
-#include <cstddef>
-#include <cstdlib>
 // #include <cstdint>
 #include <cstring>
 
-using std::vector;
-
 #include <curl/curl.h>
 
-#include "S3Common.h"
+#include "s3common.h"
+
+using std::vector;
 
 struct Range {
     /* data */

gpAux/extensions/gps3ext/src/S3Downloader_test.cpp → gpAux/extensions/gps3ext/src/s3downloader_test.cpp

-#include "S3Downloader.cpp"
+#include "s3downloader.cpp"
 #include "gtest/gtest.h"
 
 volatile bool QueryCancelPending = false;
@@ -40,7 +40,7 @@ TEST(OffsetMgr, reset) {
     delete o;
 }
 
-#ifdef FAKETEST
+#ifdef FAKE_TEST
 #define HOSTSTR "localhost"
 #define BUCKETSTR "metro.pivotal.io"
 TEST(ListBucket, fake) {
@@ -49,26 +49,15 @@ TEST(ListBucket, fake) {
     ASSERT_NE(r, (void *)NULL);
     EXPECT_EQ(16, r->contents.size());
 
-    /*
-     * char urlbuf[256];
-     * vector<BucketContent *>::iterator i;
-     * for (i = r->contents.begin(); i != r->contents.end(); i++) {
-     *     BucketContent *p = *i;
-     *     sprintf(urlbuf, "http://%s/%s/%s", HOSTSTR, BUCKETSTR,
-     *             p->Key().c_str());
-     *     printf("%s, %llu\n", urlbuf, p->Size());
-     * }
-     */
-
     delete r;
 }
-#endif  // FAKETEST
+#endif  // FAKE_TEST
 
 #define S3REGION "us-west-2"
 #define S3BUCKET "s3test.pivotal.io"
 #define S3PREFIX "dataset1/small17"
 
-#ifdef AWSTEST
+#ifdef AWS_TEST
 
 TEST(ListBucket, s3) {
     InitConfig("test/s3.conf", "default");
@@ -125,7 +114,7 @@ TEST(ListBucket, s3_5120) {
     delete r;
 }
 
-#endif  // AWSTEST
+#endif  // AWS_TEST
 
 void DownloadTest(const char *url, const char *region, uint64_t file_size,
                   const char *md5_str, uint8_t thread_num, uint64_t chunk_size,
@@ -166,7 +155,7 @@ void DownloadTest(const char *url, const char *region, uint64_t file_size,
     free(buf);
 }
 
-#ifdef FAKETEST
+#ifdef FAKE_TEST
 void HTTPDownloaderTest(const char *url, uint64_t file_size,
                         const char *md5_str, uint8_t thread_num,
                         uint64_t chunk_size, uint64_t buffer_size) {
@@ -215,11 +204,13 @@ TEST(HTTPDownloader, random_parameters_1M) {
                        571 * 1023);
 }
 
+#ifdef BIG_FILE_TEST
 TEST(HTTPDownloader, random_parameters_1G) {
     HTTPDownloaderTest("http://localhost/testdata/1G", 1073741824,
                        "a2cb2399eb8bc97084aed673e5d09f4d", 9, 42 * 1023,
                        513 * 1025 * 37);
 }
+#endif
 
 TEST(HTTPDownloader, random_parameters_8M) {
     HTTPDownloaderTest("http://localhost/testdata/8M", 8388608,
@@ -238,9 +229,9 @@ TEST(HTTPDownloader, random_parameters_256M) {
                        "9cb7c287fdd5a44378798d3e75d2d2a6", 3, 1 * 10523,
                        77 * 879);
 }
-#endif  // FAKETEST
+#endif  // FAKE_TEST
 
-#ifdef AWSTEST
+#ifdef AWS_TEST
 void S3DwonloadTest(const char *url, const char *region, uint64_t file_size,
                     const char *md5_str, uint8_t thread_num,
                     uint64_t chunk_size, uint64_t buffer_size) {
@@ -282,6 +273,7 @@ TEST(S3Downloader, httpssimple) {
         1024 * 1024, 65536);
 }
 
+#ifdef BIG_FILE_TEST
 TEST(S3Downloader, bigfile) {
     // InitLog();
     // s3ext_loglevel = EXT_DEBUG;
@@ -293,4 +285,6 @@ TEST(S3Downloader, bigfile) {
         "us-west-2", 4664425994, "f5811ad92c994f1d6913d5338575fe38", 4,
         64 * 1024 * 1024, 65536);
 }
-#endif  // AWSTEST
+#endif
+
+#endif  // AWS_TEST

gpAux/extensions/gps3ext/src/S3Log.cpp → gpAux/extensions/gps3ext/src/s3log.cpp

-#include <cstring>
-#include <sys/socket.h>
-#include <sys/types.h>
 #include <arpa/inet.h>
-#include <string>
 #include <errno.h>
-#include <unistd.h>
-#include <cstdio>
-#include <cstdarg>
-#include <sys/un.h>
-#include <sys/stat.h>
 #include <fcntl.h>
-
 #include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+#include <cstdarg>
+#include <cstdio>
+#include <cstring>
+#include <string>
 // #include <thread>
-
-#include <vector>
 #include <memory>
+#include <vector>
+#include "s3conf.h"
+#include "s3log.h"
+#include "s3utils.h"
+
 using std::vector;
 // using std::shared_ptr;
 // using std::make_shared;
-#include "utils.h"
-
-#include "S3Log.h"
-#include "gps3conf.h"
 
 #ifndef UNIX_PATH_MAX
 #define UNIX_PATH_MAX 108
 #endif
 
-#ifndef DEBUGS3
+#ifndef DEBUG_S3
 extern "C" {
 void write_log(const char* fmt, ...) __attribute__((format(printf, 1, 2)));
 }
@@ -40,7 +38,7 @@ void _LogMessage(const char* fmt, va_list args) {
     int len = vsnprintf(buf, sizeof(buf), fmt, args);
     if (len >= 1024) len = 1023;
     buf[len] = 0;
-#ifdef DEBUGS3
+#ifdef DEBUG_S3
     fprintf(stderr, "%s\n", buf);
 #else
     write_log("%s", buf);

gpAux/extensions/gps3ext/src/S3Log.h → gpAux/extensions/gps3ext/src/s3log.h

@@ -5,7 +5,8 @@
 #include <cstdio>
 
 #include <pthread.h>
-#include "gps3conf.h"
+
+#include "s3conf.h"
 
 // log level
 enum LOGLEVEL { EXT_FATAL, EXT_ERROR, EXT_WARNING, EXT_INFO, EXT_DEBUG };

gpAux/extensions/gps3ext/src/S3Log_test.cpp → gpAux/extensions/gps3ext/src/s3log_test.cpp

+#include "s3log.cpp"
 #include "gtest/gtest.h"
-#include "S3Log.cpp"
 
-TEST(logger, simple) {
+TEST(Logger, simple) {
     InitConfig("test/s3test.conf", "");
     InitLog();
     fprintf(stderr, "Hello, log type %d\n", s3ext_logtype);
@@ -9,7 +9,7 @@ TEST(logger, simple) {
     S3ERROR("Hello, ERROR");
 }
 
-TEST(logger, getstr) {
+TEST(Logger, getstr) {
     EXPECT_EQ(EXT_DEBUG, getLogLevel("DEBUG"));
     EXPECT_EQ(EXT_WARNING, getLogLevel("WARNING"));
     EXPECT_EQ(EXT_INFO, getLogLevel("INFO"));

gpAux/extensions/gps3ext/src/testq.cpp → gpAux/extensions/gps3ext/src/s3testq.cpp

-#include <unistd.h>
+#include <iostream>
+#include <memory>
+#include <vector>
+
+#include <fcntl.h>
 #include <pthread.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 
-#include <memory>
-#include <iostream>
+#include "s3utils.h"
 
+using std::cout;
 using std::shared_ptr;
 using std::make_shared;
-using std::cout;
-
-#include <vector>
 using std::vector;
 
-#include "utils.h"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
 // According to S3, index value is 1 ~ 10000
 // so 0 means initialization and -1 means finish / error here
 

gpAux/extensions/gps3ext/src/S3Uploader.cpp → gpAux/extensions/gps3ext/src/s3uploader.cpp

-#include "S3Uploader.h"
-#include "S3Common.h"
-#include "S3Downloader.h"
-
-#include <algorithm>  // std::min
-#include <sstream>
+#include <algorithm>
 #include <iostream>
-
-#include "utils.h"
+#include <sstream>
+#include <string>
 
 #define __STDC_FORMAT_MACROS
-#include <inttypes.h>
-
 #include <fcntl.h>
+#include <inttypes.h>
 #include <sys/stat.h>
 
-#include <string>
+#include "s3common.h"
+#include "s3downloader.h"
+#include "s3uploader.h"
+#include "s3utils.h"
+
 using std::string;
 using std::stringstream;
 
-#ifdef S3_UPLOAD_DEBUG_CURL
+#ifdef DEBUG_S3_CURL
 struct debug_data {
     char trace_ascii; /* 1 or 0 */
 };
@@ -199,8 +197,9 @@ const char *GetUploadId(const char *host, const char *bucket,
     CURL *curl = curl_easy_init();
     if (curl) {
         curl_easy_setopt(curl, CURLOPT_URL, url.str().c_str());
-
-        // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#if DEBUG_S3_CURL
+        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#endif
         curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L);
 
         curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&xml);
@@ -321,7 +320,9 @@ const char *PartPutS3Object(const char *host, const char *bucket,
         /* HTTP PUT please */
         curl_easy_setopt(curl, CURLOPT_PUT, 1L);
 
-        // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#if DEBUG_S3_CURL
+        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+#endif
         curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L);
 
         curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)&header_data);
@@ -424,7 +425,7 @@ bool CompleteMultiPutS3(const char *host, const char *bucket,
     XMLInfo xml;
     xml.ctxt = NULL;
 
-#ifdef S3_UPLOAD_DEBUG_CURL
+#ifdef DEBUG_S3_CURL
     struct debug_data config;
     config.trace_ascii = 1;
 #endif
@@ -486,7 +487,7 @@ bool CompleteMultiPutS3(const char *host, const char *bucket,
 
     CURL *curl = curl_easy_init();
     if (curl) {
-#ifdef S3_UPLOAD_DEBUG_CURL
+#ifdef DEBUG_S3_CURL
         curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace_debug_data);
         curl_easy_setopt(curl, CURLOPT_DEBUGDATA, &config);
 #endif
@@ -505,7 +506,7 @@ bool CompleteMultiPutS3(const char *host, const char *bucket,
            */
         curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)body_size);
 
-#ifdef S3_UPLOAD_DEBUG_CURL
+#ifdef DEBUG_S3_CURL
         curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
 #endif
         curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L);

gpAux/extensions/gps3ext/src/S3Uploader.h → gpAux/extensions/gps3ext/src/s3uploader.h

 #ifndef __S3UPLOADER_H__
 #define __S3UPLOADER_H__
 
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <pthread.h>
-
+#include <cstddef>
+#include <cstdlib>
 #include <iostream>
 #include <map>
 #include <string>
 #include <vector>
-
-#include <cstddef>
-#include <cstdlib>
 // #include <cstdint>
 #include <cstring>
 
-using std::vector;
+#include <fcntl.h>
+#include <pthread.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include <curl/curl.h>
 
-#include "S3Common.h"
+#include "s3common.h"
+
+using std::vector;
 
 struct Uploader {
     Uploader();

gpAux/extensions/gps3ext/src/S3Uploader_test.cpp → gpAux/extensions/gps3ext/src/s3uploader_test.cpp

+#include "s3uploader.cpp"
 #include "gtest/gtest.h"
-#include "S3Uploader.cpp"
 
-#ifdef AWSTEST
+#ifdef AWS_TEST
 
 #define S3HOST "s3-us-west-2.amazonaws.com"
 #define S3BUCKET "s3test.pivotal.io/upload_test"
@@ -122,4 +122,4 @@ TEST(Uploader, get_upload_id_spaces_directories) {
 }
 */
 
-#endif  // AWSTEST
+#endif  // AWS_TEST

gpAux/extensions/gps3ext/src/utils.cpp → gpAux/extensions/gps3ext/src/s3utils.cpp

-#include <sys/types.h>
-#include <sys/socket.h>
-#include <arpa/inet.h>
-#include <cstring>
+#include <algorithm>
 #include <cstdarg>
 #include <cstdio>
-#include <unistd.h>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
+#include <string>
 
-#include <time.h>
-#include <string.h>
-#include <stdio.h>
+#include <arpa/inet.h>
 #include <stdbool.h>
-#include <openssl/sha.h>
-#include <openssl/des.h>
-#include <openssl/hmac.h>
-#include <openssl/bio.h>
-#include <openssl/evp.h>
-#include <openssl/buffer.h>
 #include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
 
 #include <curl/curl.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+#include <openssl/des.h>
+#include <openssl/evp.h>
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
 
-#include "utils.h"
+#include "s3utils.h"
 
-#include <string>
-#include <sstream>
-#include <algorithm>
 using std::string;
 
-#include <iomanip>
-
-#ifndef DEBUGS3
+#ifndef DEBUG_S3
 extern "C" {
 void write_log(const char *fmt, ...) __attribute__((format(printf, 1, 2)));
 }
@@ -85,33 +84,6 @@ bool trim(char *out, const char *in, const char *trimed) {
     return true;
 }
 
-//! returning value is malloced
-char *Base64Encode(const char *buffer,
-                   size_t length) {  // Encodes a binary safe base 64 string
-    BIO *bio, *b64;
-    BUF_MEM *bufferPtr;
-    char *ret;
-    b64 = BIO_new(BIO_f_base64());
-    bio = BIO_new(BIO_s_mem());
-    bio = BIO_push(b64, bio);
-
-    BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL);  // Ignore newlines - write
-                                                 // everything in one line
-    BIO_write(bio, buffer, length);
-    BIO_flush(bio);
-    BIO_get_mem_ptr(bio, &bufferPtr);
-
-    ret = (char *)malloc(bufferPtr->length + 1);
-    if (ret) {
-        memcpy(ret, bufferPtr->data, bufferPtr->length);
-        ret[bufferPtr->length] = 0;
-    }
-
-    BIO_set_close(bio, BIO_NOCLOSE);
-    BIO_free_all(bio);
-    return ret;  // s
-}
-
 // not returning the normal hex result, might have '\0'
 bool sha1hmac(const char *str, unsigned char out_hash[20], const char *secret,
               int secret_len) {
@@ -222,6 +194,12 @@ CURL *CreateCurlHandler(const char *path) {
     return curl;
 }
 
+/*
+ * It would be more efficient to use a variation of KMP to
+ * benefit from the failure function.
+ * - Algorithm inspired by James Kanze.
+ * - http://stackoverflow.com/questions/20406744/
+ */
 size_t find_Nth(const string &str,  // where to work
                 unsigned N,         // N'th ocurrence
                 const string &find  // what to 'find'
@@ -240,12 +218,6 @@ size_t find_Nth(const string &str,  // where to work
         ++i;
     }
     return pos;
-    /**
-It would be more efficient to use a variation of KMP to
-benefit from the failure function.
-- Algorithm inspired by James Kanze.
-- http://stackoverflow.com/questions/20406744/
-    */
 }
 
 MD5Calc::MD5Calc() {
@@ -296,7 +268,7 @@ uint64_t DataBuffer::append(const char *buf, uint64_t len) {
 Config::Config(string filename) : _conf(NULL) {
     if (filename != "") this->_conf = ini_load(filename.c_str());
     if (this->_conf == NULL) {
-#ifndef DEBUGS3
+#ifndef DEBUG_S3
         write_log("Failed to load config file\n");
 #endif
     }

gpAux/extensions/gps3ext/src/utils.h → gpAux/extensions/gps3ext/src/s3utils.h

@@ -4,13 +4,16 @@
 #include <cstdio>
 #include <cstdlib>
 // #include <cstdint>
+#include <stdint.h>
+#include <sys/types.h>
 #include <cstring>
 #include <string>
-#include <openssl/sha.h>
+
 #include <openssl/md5.h>
+#include <openssl/sha.h>
 
 #include "ini.h"
-#include "S3Log.h"
+#include "s3log.h"
 
 using std::string;
 
@@ -21,9 +24,6 @@ void _tolower(char* buf);
 
 bool trim(char* out, const char* in, const char* trimed = " \t\r\n");
 
-//! return value is malloced
-char* Base64Encode(const char* buffer, size_t length);
-
 bool sha1hmac(const char* str, unsigned char out_hash[20], const char* secret,
               int secret_len);
 
@@ -59,9 +59,9 @@ class MD5Calc {
 };
 
 #if 0
+#include <condition_variable>
 #include <mutex>
 #include <queue>
-#include <condition_variable>
 using std::mutex;
 using std::condition_variable;
 using std::queue;

gpAux/extensions/gps3ext/src/utils_test.cpp → gpAux/extensions/gps3ext/src/s3utils_test.cpp

+#include "s3utils.cpp"
 #include "gtest/gtest.h"
-#include "utils.cpp"
-#include <string>
 
-TEST(utils, lower) {
+TEST(Utils, lower) {
     char data[] = "aAbBcCdDEeFfGgHhIiJjKkLlMmNnOopPQqRrSsTtuUVvwWxXYyZz";
     _tolower(data);
     EXPECT_STREQ("aabbccddeeffgghhiijjkkllmmnnooppqqrrssttuuvvwwxxyyzz", data);
 }
 
 // Tests factorial of positive numbers.
-TEST(utils, trim) {
+TEST(Utils, trim) {
     char data[] = " \t\n\r  abc \r\r\n\r \t";
     char out[8] = {0};
     bool ret;
@@ -18,13 +17,12 @@ TEST(utils, trim) {
     EXPECT_STREQ("abc", out);
 }
 
-TEST(utils, time) {
+TEST(Utils, time) {
     char data[65];
     gethttpnow(data);
 }
 
-#include <curl/curl.h>
-TEST(utils, simplecurl) {
+TEST(Utils, simplecurl) {
     CURL *c = CreateCurlHandler(NULL);
     EXPECT_EQ(c, (void *)NULL);
     c = CreateCurlHandler("www.google.com");
@@ -32,7 +30,7 @@ TEST(utils, simplecurl) {
     curl_easy_cleanup(c);
 }
 
-TEST(utils, nth) {
+TEST(Utils, nth) {
     const char *teststr = "aaabbbcccaaatttaaa";
     EXPECT_EQ(find_Nth(teststr, 1, "aaa"), 0);
     EXPECT_EQ(find_Nth(teststr, 2, "aaa"), 9);
@@ -42,7 +40,7 @@ TEST(utils, nth) {
 }
 
 #define MD5TESTBUF "abcdefghijklmnopqrstuvwxyz\n"
-TEST(utils, md5) {
+TEST(Utils, md5) {
     MD5Calc m;
     m.Update(MD5TESTBUF, strlen(MD5TESTBUF));
     EXPECT_STREQ("e302f9ecd2d189fa80aac1c3392e9b9c", m.Get());
@@ -53,14 +51,7 @@ TEST(utils, md5) {
 }
 
 #define TEST_STRING "The quick brown fox jumps over the lazy dog"
-TEST(utils, base64) {
-    char *encoded = NULL;
-    EXPECT_STREQ("VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw==",
-                 encoded = Base64Encode(TEST_STRING, strlen(TEST_STRING)));
-    free(encoded);
-}
-
-TEST(utils, sha256) {
+TEST(Utils, sha256) {
     char hash_str[65] = {0};
     EXPECT_TRUE(sha256_hex(TEST_STRING, hash_str));
     EXPECT_STREQ(
@@ -68,7 +59,7 @@ TEST(utils, sha256) {
         hash_str);
 }
 
-TEST(utils, sha1hmac) {
+TEST(Utils, sha1hmac) {
     char hash_hex[41] = {0};
 
     EXPECT_TRUE(sha1hmac_hex(TEST_STRING, (char *)hash_hex, "key", 3));
@@ -76,7 +67,7 @@ TEST(utils, sha1hmac) {
     EXPECT_STREQ("de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", hash_hex);
 }
 
-TEST(utils, sha256hmac) {
+TEST(Utils, sha256hmac) {
     char hash_str[65] = {0};
     EXPECT_TRUE(sha256hmac_hex(TEST_STRING, hash_str, "key", 3));
     EXPECT_STREQ(
@@ -160,7 +151,7 @@ TEST(databuffer, simple) {
 }
 #endif
 
-TEST(utils, Config) {
+TEST(Utils, Config) {
     Config c("test/s3test.conf");
     EXPECT_STREQ(c.Get("configtest", "config1", "aaaaaa").c_str(), "abcdefg");
     EXPECT_STREQ(c.Get("configtest", "config2", "tttt").c_str(), "12345");
@@ -193,7 +184,7 @@ TEST(utils, Config) {
     EXPECT_STREQ(str, "aaaaa");
 }
 
-TEST(utils, UriCoding) {
+TEST(Utils, UriCoding) {
     string src1 = "This is a simple & short test.";
     string src2 = "$ & < > ? ; # : = , \" ' ~ + %-_";
     string src3 = "! * ' ( ) ; : @ & = + $ , / ? % # [ ]";
@@ -224,3 +215,28 @@ TEST(utils, UriCoding) {
     EXPECT_STREQ(src3.c_str(), uri_decode(dst3).c_str());
     EXPECT_STREQ(src4.c_str(), uri_decode(dst4).c_str());
 }
+
+TEST(Utils, find_replace) {
+    string str1 = "This is a simple & short test.";
+
+    find_replace(str1, "simple", "");
+    EXPECT_STREQ("This is a  & short test.", str1.c_str());
+
+    find_replace(str1, "short ", "");
+    EXPECT_STREQ("This is a  & test.", str1.c_str());
+
+    find_replace(str1, "test.", "");
+    EXPECT_STREQ("This is a  & ", str1.c_str());
+
+    find_replace(str1, "This", "");
+    EXPECT_STREQ(" is a  & ", str1.c_str());
+
+    find_replace(str1, "is a", "abcdefghijklmn");
+    EXPECT_STREQ(" abcdefghijklmn  & ", str1.c_str());
+
+    find_replace(str1, " a", "a");
+    EXPECT_STREQ("abcdefghijklmn  & ", str1.c_str());
+
+    find_replace(str1, "abc", "abcabc");
+    EXPECT_STREQ("abcabcdefghijklmn  & ", str1.c_str());
+}

gpAux/extensions/gps3ext/src/S3ExtWrapper.cpp → gpAux/extensions/gps3ext/src/s3wrapper.cpp

-#include "S3ExtWrapper.h"
-#include "utils.h"
-#include "gps3conf.h"
+#include <sstream>
+#include <string>
+
 #include "gps3ext.h"
-#include "S3Log.h"
+#include "s3conf.h"
+#include "s3log.h"
+#include "s3utils.h"
+#include "s3wrapper.h"
 
-#include <string>
-#include <sstream>
-using std::stringstream;
 using std::string;
+using std::stringstream;
 
 // invoked by s3_import(), need to be exception safe
 S3ExtBase *CreateExtWrapper(const char *url) {
@@ -190,13 +191,11 @@ bool S3Reader::TransferData(char *data, uint64_t &len) {
 // invoked by s3_import(), need to be exception safe
 bool S3Reader::Destroy() {
     try {
-        // reset filedownloader
         if (this->filedownloader) {
             this->filedownloader->destroy();
             delete this->filedownloader;
         }
 
-        // Free keylist
         if (this->keylist) {
             delete this->keylist;
         }

gpAux/extensions/gps3ext/src/S3ExtWrapper.h → gpAux/extensions/gps3ext/src/s3wrapper.h

 #ifndef __S3_EXT_WRAPPER__
 #define __S3_EXT_WRAPPER__
 
-#include "S3Downloader.h"
-#include "S3Uploader.h"
 #include <string>
 
+#include "s3downloader.h"
+#include "s3uploader.h"
+
 using std::string;
 
 class S3ExtBase {

gpAux/extensions/gps3ext/src/S3ExtWrapper_test.cpp → gpAux/extensions/gps3ext/src/s3wrapper_test.cpp

+#include "s3wrapper.cpp"
 #include "gtest/gtest.h"
-#include "S3ExtWrapper.cpp"
 
 class S3Reader_fake : public S3Reader {
    public:
@@ -18,13 +18,11 @@ S3Reader_fake::S3Reader_fake(string url) : S3Reader(url) {}
 S3Reader_fake::~S3Reader_fake() {}
 
 bool S3Reader_fake::Destroy() {
-    // reset filedownloader
     if (this->filedownloader) {
         this->filedownloader->destroy();
         delete this->filedownloader;
     }
 
-    // Free keylist
     if (this->keylist) {
         delete this->keylist;
     }
@@ -44,9 +42,7 @@ bool S3Reader_fake::Init(int segid, int segnum, int chunksize) {
         S3ERROR("validate url fail %s\n", this->url.c_str());
     }
 
-    // TODO: As separated function for generating url
     this->keylist = ListBucket_FakeHTTP("localhost", this->bucket.c_str());
-    // this->keylist = ListBucket_FakeHTTP("localhost", "metro.pivotal.io");
 
     if (!this->keylist) {
         return false;
@@ -196,7 +192,7 @@ TEST(ExtWrapper, ValidateURL_apnortheast21) {
     delete myData;
 }
 
-#ifdef AWSTEST
+#ifdef AWS_TEST
 
 TEST(ExtWrapper, normal_region_default) {
     ExtWrapperTest(
@@ -228,6 +224,7 @@ TEST(ExtWrapper, normal_2segs) {
         64 * 1024, "db05de0ec7e0808268e2363d3572dc7f", 1, 2, 64 * 1024 * 1024);
 }
 
+#ifdef BIG_FILE_TEST
 TEST(ExtWrapper, normal_3segs) {
     ExtWrapperTest(
         "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/"
@@ -280,10 +277,11 @@ TEST(ExtWrapper, normal2_3segs) {
         "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/normal/",
         64 * 1024, "00675684b6d6697571f22baaf407c6df", 2, 3, 64 * 1024 * 1024);
 }
+#endif
 
-#endif  // AWSTEST
+#endif  // AWS_TEST
 
-#ifdef FAKETEST
+#ifdef FAKE_TEST
 
 TEST(FakeExtWrapper, simple) {
     ExtWrapperTest("http://localhost/metro.pivotal.io/", 64 * 1024,
@@ -317,9 +315,11 @@ TEST(FakeExtWrapper, normal_4segs) {
                    "dceb001d03d54d61874d27c9f04596b1", 3, 4, 64 * 1024 * 1024);
 }
 
+#ifdef BIG_FILE_TEST
 TEST(FakeExtWrapper, bigfile) {
     ExtWrapperTest("http://localhost/bigfile/", 64 * 1024,
                    "83c7ab787e3f1d1e7880dcae954ab4a4", 0, 1, 64 * 1024 * 1024);
 }
+#endif
 
-#endif  // FAKETEST
+#endif  // FAKE_TEST

gpAux/extensions/gps3ext/test/mvp_test_1.1.sql → gpAux/extensions/gps3ext/test/regress_1.01.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-        low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal config=/home/gpadmin/s3.conf') FORMAT 'csv';
+        low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
 
 SELECT count(*) FROM s3example;
 SELECT sum(open) FROM s3example;

gpAux/extensions/gps3ext/test/mvp_test_1.2.sql → gpAux/extensions/gps3ext/test/regress_1.02.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-        low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5;
+        low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
 SELECT gp_read_error_log('s3example');

gpAux/extensions/gps3ext/test/mvp_test_1.3.sql → gpAux/extensions/gps3ext/test/regress_1.03.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/mvp_test_1.4.sql → gpAux/extensions/gps3ext/test/regress_1.04.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/emptyfile config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
 SELECT gp_read_error_log('s3example');

gpAux/extensions/gps3ext/test/mvp_test_1.5.sql → gpAux/extensions/gps3ext/test/regress_1.05.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/oneline config=/home/gpadmin/s3.conf') format 'csv';
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/oneline/ config=/home/gpadmin/s3.conf') format 'csv';
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/mvp_test_1.6.sql → gpAux/extensions/gps3ext/test/regress_1.06.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
 SELECT gp_read_error_log('s3example');

gpAux/extensions/gps3ext/test/mvp_test_1.7.sql → gpAux/extensions/gps3ext/test/regress_1.07.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 -- create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 100 PERCENT;
-create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
+create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/mvp_test_1.8.sql → gpAux/extensions/gps3ext/test/regress_1.08.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 -- create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 100 PERCENT;
-create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile_correct_data config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
+create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile_correct_data/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/mvp_test_1.9.sql → gpAux/extensions/gps3ext/test/regress_1.09.sql

@@ -26,7 +26,7 @@ CREATE TABLE stock (date text, time text, open float, high float, low float, vol
 INSERT INTO stock (date, time, open, high, low, volume) VALUES (03/08/2011, 094022, 35.25, 35.24, 35.29, 20001);
 
 create READABLE external table ext_stock000 (date text, time text, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal config=/home/gpadmin/s3.conf') FORMAT 'csv';
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
 
 ALTER TABLE stock ADD PARTITION stock000 START (0) INCLUSIVE END (10000) EXCLUSIVE;
 

gpAux/extensions/gps3ext/test/regress_1.10.sql

+-- ========
+-- PROTOCOL
+-- ========
+
+-- create the database functions
+CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
+        '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
+
+-- declare the protocol name along with in/out funcs
+CREATE PROTOCOL s3 (
+        readfunc  = read_from_s3
+);
+
+-- Check out the catalog table
+select * from pg_extprotocol;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-ap-northeast-2.amazonaws.com/apnortheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-ap-southeast-1.amazonaws.com/apsoutheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-ap-southeast-2.amazonaws.com/apsoutheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-eu-west-1.amazonaws.com/euwest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- ========
+
+drop external table s3example;
+create READABLE external table s3example (date text, time text, open float, high float,
+        low float, volume int) location('s3://s3-us-west-1.amazonaws.com/uswest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
+
+\d s3example
+
+SELECT count(*) FROM s3example;
+SELECT sum(open) FROM s3example;
+SELECT avg(open) FROM s3example;
+
+-- =======
+-- CLEANUP
+-- =======
+DROP EXTERNAL TABLE s3example;
+
+DROP PROTOCOL s3;

gpAux/extensions/gps3ext/test/mvp_test_2.1.sql → gpAux/extensions/gps3ext/test/regress_2.01.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date float, time float, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal config=/home/gpadmin/s3.conf') format 'csv';
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') format 'csv';
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/mvp_test_2.2.sql → gpAux/extensions/gps3ext/test/regress_2.02.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://neverland.amazonaws.com/wherever/smalldata config=/home/gpadmin/s3.conf') format 'csv';
+	low float, volume int) location('s3://neverland.amazonaws.com/wherever/smalldata/ config=/home/gpadmin/s3.conf') format 'csv';
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/mvp_test_2.3.sql → gpAux/extensions/gps3ext/test/regress_2.03.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal') format 'csv';
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/') format 'csv';
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/mvp_test_2.4.sql → gpAux/extensions/gps3ext/test/regress_2.04.sql

@@ -16,7 +16,7 @@ select * from pg_extprotocol;
 
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header config=/home/gpadmin/s3.conf') format 'csv';
+	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv';
 
 SELECT count(*) FROM s3example;
 

gpAux/extensions/gps3ext/test/s3test_1.sql

--- ========
--- PROTOCOL
--- ========
-
--- create the database functions
-CREATE OR REPLACE FUNCTION write_to_s3() RETURNS integer AS
-   '$libdir/gps3ext.so', 's3_export' LANGUAGE C STABLE;
-CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
-    '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-
--- declare the protocol name along with in/out funcs
-CREATE PROTOCOL s3ext (
-    readfunc  = read_from_s3,
-    writefunc = write_to_s3
-);
-
--- Check out the catalog table
-select * from pg_extprotocol;
-
-drop table example;
-create table example (id serial, name text) distributed by (id);
-insert into example (name) values ('ssf');
-insert into example (name) values ('abc');
-insert into example (name) values ('deefe3');
-insert into example (name) values ('e90jg');
-insert into example (name) values ('br t34');
-insert into example (name) values ('3r3r');
-insert into example (name) values ('ccfee4');
-insert into example (name) values ('hbafe');
-
--- create the external table with this protocol:
---   Use a url that the protocol knows how to parse later (you make it whatever you want)
-CREATE WRITABLE EXTERNAL TABLE ext_w(like example)
-    LOCATION('s3://demotextfile.txt')
-FORMAT 'text'
-DISTRIBUTED BY (id);
-
-CREATE READABLE EXTERNAL TABLE ext_r(like example)
-    LOCATION('s3://demotextfile.txt')
-FORMAT 'text';
-
--- Use the external tables
-INSERT into ext_w select * from example;
-
-SELECT * FROM ext_r;
-
--- =======
--- CLEANUP
--- =======
-DROP EXTERNAL TABLE ext_r;
-DROP EXTERNAL TABLE ext_w;
-
-DROP PROTOCOL s3ext;

gpAux/extensions/gps3ext/test/s3test_2.sql

--- ========
--- PROTOCOL
--- ========
-
--- create the database functions
-CREATE OR REPLACE FUNCTION write_to_s3() RETURNS integer AS
-	'$libdir/gps3ext.so', 's3_export' LANGUAGE C STABLE;
-CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
-	'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-
--- declare the protocol name along with in/out funcs
-CREATE PROTOCOL s3ext (
-	readfunc  = read_from_s3,
-	writefunc = write_to_s3
-);
-
--- Check out the catalog table
-select * from pg_extprotocol;
-
-drop external table s3example;
-create READABLE external table s3example (date text, time text, open float, high float,
-	low float, volume int) location('s3://demotextfile.txt') FORMAT 'csv';
-
-SELECT count(*) FROM s3example;
-
--- =======
--- CLEANUP
--- =======
-DROP EXTERNAL TABLE s3example;
-
-DROP PROTOCOL s3ext;

gpAux/extensions/gps3ext/test/s3test_3.sql

--- ========
--- PROTOCOL
--- ========
-
--- create the database functions
-CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
-	'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-
--- declare the protocol name along with in/out funcs
-CREATE PROTOCOL s3 (
-	readfunc  = read_from_s3
-);
-
--- Check out the catalog table
-select * from pg_extprotocol;
-
-drop external table s3example;
-create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/normal/ config=/home/gpadmin/data/gps3ext/test/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 100 PERCENT;
--- select * into s3data from s3example;
-SELECT count(*) FROM s3example;
-
--- =======
--- CLEANUP
--- =======
-DROP EXTERNAL TABLE s3example;
-
-DROP PROTOCOL s3;

gpAux/extensions/gps3ext/test/s3test_4.sql

--- ========
--- PROTOCOL
--- ========
-
--- create the database functions
-CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
-	'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-
--- declare the protocol name along with in/out funcs
-CREATE PROTOCOL s3 (
-	readfunc  = read_from_s3
-);
-
--- Check out the catalog table
-select * from pg_extprotocol;
-
-drop external table s3example;
-
-create READABLE external table s3example (date text, time text, open float, high float,
-       low float, volume int) location('s3://s3-us-west-2.amazonaws.com/metro.pivotal.io/baddata config=/home/gpadmin/work/bootgpdb/data/gpseg0/s3/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
-SELECT count(*) FROM s3example;
-
--- =======
--- CLEANUP
--- =======
-DROP EXTERNAL TABLE s3example;
-
-DROP PROTOCOL s3;

gpAux/extensions/gps3ext/test/s3test_5.sql

--- ========
--- PROTOCOL
--- ========
-
--- create the database functions
-CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
-	'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-
--- declare the protocol name along with in/out funcs
-CREATE PROTOCOL s3 (
-	readfunc  = read_from_s3
-);
-
--- Check out the catalog table
-select * from pg_extprotocol;
-
-drop external table s3example;
-create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/airlinedata2.csv config=/home/gpadmin/data/gps3ext/test/s3.conf') format 'csv' segment REJECT LIMIT 100 PERCENT;
--- select * into s3data from s3example;
-SELECT count(*) FROM s3example;
-
--- =======
--- CLEANUP
--- =======
-DROP EXTERNAL TABLE s3example;
-
-DROP PROTOCOL s3;