Add gpdb client test cases on windows pipeline (#7926)

1) Run TEST_REMOTE.py of gpload
2) Run psql SSL, gpfdist Windows pipe testing
Co-authored-by: NPeifeng Qiu <pqiu@pivotal.io>
Co-authored-by: NTingfang Bao <bbao@pivotal.io>
Co-authored-by: NXiaoran Wang <xiwang@pivotal.io>
Co-authored-by: NXiaodong Huo <xhuo@pivotal.io>

concourse/pipelines/gen_pipeline.py

@@ -70,6 +70,7 @@ JOBS_THAT_SHOULD_NOT_BLOCK_RELEASE = (
     [
         'compile_gpdb_binary_swap_centos6',
         'compile_gpdb_clients_windows',
+        'test_gpdb_clients_windows',
         'icw_gporca_centos6_gpos_memory',
         'walrep_2',
         'madlib_build_gppkg',

concourse/pipelines/templates/gpdb-tpl.yml

@@ -54,6 +54,7 @@ groups:
 {% endif %}
 {% if "win" in os_types %}
   - compile_gpdb_clients_windows
+  - test_gpdb_clients_windows
 {% endif %}
 {% if "ICW" in test_sections %}
   ## --------------------------------------------------------------------
@@ -155,6 +156,7 @@ groups:
 {% endif %}
 {% if "win" in os_types %}
   - compile_gpdb_clients_windows
+  - test_gpdb_clients_windows
 {% endif %}
 
 {% if "ICW" in test_sections %}
@@ -615,13 +617,39 @@ resources:
 
 {% endif %}
 {% if "win" in os_types %}
+- name: terraform_windows
+  type: terraform
+  source:
+    env:
+      AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
+      AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
+      GOOGLE_CREDENTIALS: {{google-service-account-key}}
+    vars:
+      project: {{google-project-id}}
+      region: {{google-region}}
+      region_zone: {{google-zone}}
+    storage:
+      access_key_id: {{tf-machine-access-key-id}}
+      secret_access_key: {{tf-machine-secret-access-key}}
+      region_name: {{aws-region}}
+      # This is not parameterized, on purpose. All tfstates will go to this spot,
+      # and different teams will place there clusters' tfstate files under different paths
+      bucket: gpdb5-pipeline-dynamic-terraform
+      bucket_path: gp-clients-windows-tests-google/
+
 - name: bin_gpdb_clients_windows
   type: gcs
   source:
     bucket: ((gcs-bucket-intermediates))
     json_key: ((concourse-gcs-resources-service-account-key))
-    versioned_file: ((pipeline-name))/bin_gpdb_clients_windows/bin_gpdb_clients_windows.msi
+    versioned_file: ((pipeline-name))/bin_gpdb_clients_windows/greenplum-clients-x86_64.tar.gz
 
+- name: bin_gpdb_clients_windows_rc
+  type: gcs
+  source:
+    bucket: ((gcs-bucket))
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: clients/published/master/greenplum-clients-(.*)-x86_64.msi
 {% endif %}
 {% if "AA" in test_sections %}
 - name: madlib_src
@@ -982,7 +1010,44 @@ jobs:
       REMOTE_KEY: {{remote_win_key_build}}
   - put: bin_gpdb_clients_windows
     params:
-      file: "gpdb_artifacts/greenplum-clients-*-x86_64.msi"
+      file: "gpdb_artifacts/greenplum-clients-x86_64.tar.gz"
+
+- name: test_gpdb_clients_windows
+  plan:
+  - aggregate:
+      - get: ccp_src
+      - get: gpdb_src
+        trigger: true
+        passed: [compile_gpdb_clients_windows]
+      - get: gpdb6-centos7-test
+      - get: bin_gpdb_clients_windows
+        passed: [compile_gpdb_clients_windows]
+      - get: bin_gpdb
+        resource: bin_gpdb_centos7_rc
+  - put: terraform_windows
+    params:
+      action: create
+      delete_on_failure: true
+      generate_random_name: true
+      terraform_source: ccp_src/gpdb_windows_client_test/
+  - task: run_tests
+    file: gpdb_src/concourse/tasks/windows_remote_test.yml
+    image: gpdb6-centos7-test
+    params:
+      REMOTE_PORT: {{remote_win_port_test}}
+      REMOTE_USER: {{remote_win_user_test}}
+      REMOTE_KEY: {{remote_win_key_test}}
+  ensure:
+    put: terraform_windows
+    params:
+      action: destroy
+      env_name_file: terraform_windows/name
+      terraform_source: ccp_src/gpdb_windows_client_test/
+      vars:
+        aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
+        aws_ebs_volume_type: standard
+    get_params:
+      action: destroy
 {% endif %}
 
 {% if "ICW" in test_sections %}
@@ -1831,6 +1896,7 @@ jobs:
       - resource_group_centos7
       - gpperfmon_centos6
       - gpperfmon_centos7
+      - test_gpdb_clients_windows
 {% for test in CLI_BEHAVE_TESTS %}
       - [[ test.name ]]
       - [[ test.name ]]_ubuntu18
@@ -1888,6 +1954,9 @@ jobs:
     - get: bin_gpdb_clients_ubuntu18.04
       passed:
       - compile_gpdb_ubuntu18.04
+    - get: bin_gpdb_clients_windows
+      passed:
+      - test_gpdb_clients_windows
 
 - name: Release_Candidate
   plan:
@@ -1913,6 +1982,8 @@ jobs:
     - get: bin_gpdb_clients_ubuntu18.04
       trigger: true
       passed: [gate_release_candidate_start]
+    - get: bin_gpdb_clients_windows
+      passed: [gate_release_candidate_start]
     - get: gpdb6-centos6-build
   - aggregate:
     - task: rename_rc_artifacts
@@ -1940,6 +2011,9 @@ jobs:
     - put: bin_gpdb_clients_ubuntu18.04_rc
       params:
         file: "release_candidates/clients-rc-*ubuntu18.04*.tar.gz"
+    - put: bin_gpdb_clients_windows_rc
+      params:
+        file: "release_candidates/clients-rc-*-windows_x86_64.msi"
     - put: bin_gpdb_centos6_rc_s3
       params:
         file: bin_gpdb_centos6/bin_gpdb.tar.gz

concourse/scripts/compile_gpdb_remote_windows.bash

-#! /bin/bash
+#!/bin/bash
+#shellcheck disable=2153,2087,2035,2140
+
 set -eo pipefail
 
 ROOT_DIR=$(pwd)
@@ -8,7 +10,7 @@ ROOT_DIR=$(pwd)
 # since newline doesn't work well for env variable.
 function setup_ssh_keys() {
     # Setup ssh keys
-    echo -n "$REMOTE_KEY" | base64 -d > ~/remote.key
+    echo -n "${REMOTE_KEY}" | base64 -d > ~/remote.key
     chmod 400 ~/remote.key
 
     eval "$(ssh-agent -s)"
@@ -16,7 +18,7 @@ function setup_ssh_keys() {
 
     # Scan for target server's public key, append port number
     mkdir -p ~/.ssh
-    ssh-keyscan -p "$REMOTE_PORT" "$REMOTE_HOST" > ~/.ssh/known_hosts
+    ssh-keyscan -p "${REMOTE_PORT}" "${REMOTE_HOST}" > ~/.ssh/known_hosts
 }
 
 # All the variables in this funciton are GLOBAL environment variables.
@@ -28,11 +30,12 @@ function remote_setup() {
     WORK_DIR="C:\\Users\\buildbot\\${SESSION_ID}"
     
     # Get git information from local repo(concourse gpdb_src input)
-    cd gpdb_src
-    GIT_URI=$(git config --get remote.origin.url)
-    GIT_COMMIT=$(git rev-parse HEAD)
-    GIT_TAG=$(git describe --tags --abbrev=0 | grep -E -o '[0-9]\.[0-9]+\.[0-9]+')
-    cd ..
+    pushd gpdb_src
+        GIT_URI=$(git config --get remote.origin.url)
+        GIT_COMMIT=$(git rev-parse HEAD)
+        GIT_TAG=$(git describe --tags --abbrev=0 | grep -E -o '[0-9]\.[0-9]+\.[0-9]+')
+        GPDB_VERSION=$(./getversion --short)
+    popd
 }
 
 # Since we're cloning in a different machine, maybe there's 
@@ -41,28 +44,33 @@ function remote_setup() {
 function remote_clone() {
     # Connect to remote windows server powershell environment and execute
     # the specified commands
-    ssh -A -T -p "$REMOTE_PORT" "$REMOTE_USER"@"$REMOTE_HOST" <<- EOF
-    mkdir "$WORK_DIR"
-    cd "$WORK_DIR"
-    git clone "$GIT_URI" gpdb_src
+    ssh -A -T -p "${REMOTE_PORT}" "${REMOTE_USER}"@"${REMOTE_HOST}" <<- EOF
+    mkdir "${WORK_DIR}"
+    cd "${WORK_DIR}"
+    git clone "${GIT_URI}" gpdb_src
     cd gpdb_src
-    git reset --hard "$GIT_COMMIT"
+    git reset --hard "${GIT_COMMIT}"
 EOF
 }
 
 function remote_compile() {
     # Connect to remote windows server powershell environment and execute
     # the specified commands
-    ssh -T -p "$REMOTE_PORT" "$REMOTE_USER"@"$REMOTE_HOST" <<- EOF
-    cd "$WORK_DIR\gpdb_src"
-    set WORK_DIR=$WORK_DIR
-    set GPDB_VERSION=$GIT_TAG
+    ssh -T -p "${REMOTE_PORT}" "${REMOTE_USER}"@"${REMOTE_HOST}" <<- EOF
+    cd "${WORK_DIR}\gpdb_src"
+    set WORK_DIR=${WORK_DIR}
+    set GPDB_VERSION=${GIT_TAG}
     concourse\scripts\compile_gpdb_remote_windows.bat
 EOF
 }
 
 function download() {
-    scp -P "$REMOTE_PORT" -q "$REMOTE_USER"@"$REMOTE_HOST":"$WORK_DIR/*.msi" "$ROOT_DIR/gpdb_artifacts/"
+    pushd "$ROOT_DIR/gpdb_artifacts/"
+        scp -P "${REMOTE_PORT}" -q "${REMOTE_USER}"@"${REMOTE_HOST}":"${WORK_DIR}/*.msi" ./
+        scp -P "${REMOTE_PORT}" -q "${REMOTE_USER}"@"${REMOTE_HOST}":"${WORK_DIR}/*.exe" ./
+        echo "${GPDB_VERSION}" > version
+        tar cvzf greenplum-clients-x86_64.tar.gz *
+    popd
 }
 
 # Since we are cloning and building on remote machine,
@@ -71,14 +79,14 @@ function download() {
 function cleanup() {
     # Connect to remote windows server powershell environment and execute
     # the specified commands
-    ssh -T -p "$REMOTE_PORT" "$REMOTE_USER"@"$REMOTE_HOST" <<- EOF
-    rmdir /S /Q "$WORK_DIR"
+    ssh -T -p "${REMOTE_PORT}" "${REMOTE_USER}"@"${REMOTE_HOST}" <<- EOF
+    rmdir /S /Q "${WORK_DIR}"
 EOF
 }
 
 function _main() {
 
-    if [ -z "$REMOTE_PORT" ]; then
+    if [[ -z "${REMOTE_PORT}" ]]; then
         REMOTE_PORT=22
     fi
 

concourse/scripts/compile_gpdb_remote_windows.bat

 @call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
 @echo on
 cd %WORK_DIR%\gpdb_src\src\tools\msvc
-echo "our $config = {gss => 'C:/ext', openssl => 'c:/ext', zlib => 'c:/ext'};"  >config.pl
+echo our $config = {gss =^> 'C:/ext', openssl =^> 'c:/ext', zlib =^> 'c:/ext'}; >config.pl
 
 perl build.pl client
 perl install.pl %WORK_DIR%\greenplum-db-devel client
@@ -29,3 +29,7 @@ cd %WORK_DIR%\gpdb_src\gpAux\client\install\src\windows\
 
 move *.msi %WORK_DIR%
 
+REM build window pipe client
+cd %WORK_DIR%\gpdb_src\src\bin\gpfdist\wintest\
+cl pipe_win10.cpp
+move *.exe %WORK_DIR%

concourse/scripts/ic_gpdb_remote_windows.bash

+#!/bin/bash -l
+#shellcheck disable=1090,1091,2129,2035
+
+set -eo pipefail
+
+CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "${CWDIR}/common.bash"
+
+function setup_gpadmin_user() {
+    ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash "${TEST_OS}"
+}
+
+function configure_gpdb_ssl() {
+    cp ./gpdb_src/src/test/ssl/ssl/server.crt "${MASTER_DATA_DIRECTORY}"
+    cp ./gpdb_src/src/test/ssl/ssl/server.key "${MASTER_DATA_DIRECTORY}"
+    cp ./gpdb_src/src/test/ssl/ssl/root+server_ca.crt "${MASTER_DATA_DIRECTORY}"
+    chmod 600 "${MASTER_DATA_DIRECTORY}/server.key"
+
+    PG_CONF="${MASTER_DATA_DIRECTORY}/postgresql.conf"
+    echo "ssl=on" >> "${PG_CONF}"
+    echo "ssl_ca_file='root+server_ca.crt'">> "${PG_CONF}"
+    echo "ssl_cert_file='server.crt'">> "${PG_CONF}"
+    echo "ssl_key_file='server.key'">> "${PG_CONF}"
+
+    PG_HBA="${MASTER_DATA_DIRECTORY}/pg_hba.conf"
+    echo "hostssl   all gpadmin 0.0.0.0/0   trust">> "${PG_HBA}"
+    gpstop -ar
+}
+
+# Get ssh private key from REMOTE_KEY, which is assumed to
+# be encode in base64. We can't pass the key content directly
+# since newline doesn't work well for env variable.
+function import_remote_key() {
+    echo -n "${REMOTE_KEY}" | base64 -d > ~/remote.key
+    chmod 400 ~/remote.key
+
+    eval "$(ssh-agent -s)"
+    ssh-add ~/remote.key
+
+    # Scan for target server's public key, append port number
+    # maybe run with ssh -o StrictHostKeyChecking=no?
+    mkdir -p ~/.ssh
+    ssh-keyscan -p "${REMOTE_PORT}" "${REMOTE_HOST}" > ~/.ssh/known_hosts
+}
+
+function run_remote_test() {
+    source ./gpdb_src/gpAux/gpdemo/gpdemo-env.sh
+    #restart gpdb with ssl
+    export -f configure_gpdb_ssl
+    /bin/sh -c "su gpadmin -c configure_gpdb_ssl"
+
+    scp -P "${REMOTE_PORT}" ./gpdb_src/concourse/scripts/windows_remote_test.ps1 "${REMOTE_USER}@${REMOTE_HOST}:"
+    scp -P "${REMOTE_PORT}" ./bin_gpdb_clients_windows/*.msi "${REMOTE_USER}@${REMOTE_HOST}:"
+    scp -P "${REMOTE_PORT}" ./bin_gpdb_clients_windows/*.exe "${REMOTE_USER}@${REMOTE_HOST}:"
+
+    ssh -T -R"${PGPORT}:127.0.0.1:${PGPORT}" -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_HOST}" 'powershell < windows_remote_test.ps1'
+
+    scp -P "${REMOTE_PORT}" -r ./gpdb_src/gpMgmt/bin/gpload_test/gpload2 "${REMOTE_USER}@${REMOTE_HOST}:."
+    scp -P "${REMOTE_PORT}" ./gpdb_src/src/test/regress/*.pl "${REMOTE_USER}@${REMOTE_HOST}:./gpload2"
+    scp -P "${REMOTE_PORT}" ./gpdb_src/src/test/regress/*.pm "${REMOTE_USER}@${REMOTE_HOST}:./gpload2"
+    scp -P "${REMOTE_PORT}" ./gpdb_src/concourse/scripts/ic_gpdb_remote_windows.bat "${REMOTE_USER}@${REMOTE_HOST}:"
+
+    ssh -T -R"${PGPORT}:127.0.0.1:${PGPORT}" -L8081:127.0.0.1:8081 -L8082:127.0.0.1:8082 -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_HOST}" "ic_gpdb_remote_windows.bat ${PGPORT}"
+}
+
+function create_cluster() {
+    export CONFIGURE_FLAGS="--enable-gpfdist --with-openssl"
+    yum install -y openssl-devel
+    time install_and_configure_gpdb
+    time setup_gpadmin_user
+    export WITH_MIRRORS=false
+    time make_cluster
+}
+
+function _main() {
+    if [[ -z "${REMOTE_PORT}" ]]; then
+        REMOTE_PORT=22
+    fi
+    yum install -y jq
+    REMOTE_HOST=$(jq -r '."gpdb-clients-ip"' terraform_windows/metadata)
+    export REMOTE_HOST
+
+    pushd bin_gpdb
+        mv *.tar.gz bin_gpdb.tar.gz
+    popd
+
+    pushd bin_gpdb_clients_windows
+        tar xzvf *.tar.gz
+    popd
+
+    time create_cluster
+    time import_remote_key
+    time run_remote_test
+
+    cp bin_gpdb_clients_windows/*.msi bin_gpdb_clients_windows_rc/
+    pushd bin_gpdb_clients_windows_rc
+        VERSION=$(cat ../bin_gpdb_clients_windows/version)
+        mv greenplum-clients-x86_64.msi "greenplum-clients-${VERSION}-x86_64.msi"
+    popd
+}
+
+_main "$@"

concourse/scripts/ic_gpdb_remote_windows.bat

+set PGPORT=%1
+set PGUSER=gpadmin
+set PGHOST=127.0.0.1
+
+call "C:\Program Files\Greenplum\greenplum-clients\greenplum_clients_path.bat"
+set path=%path%;C:\Program Files\curl-win64-mingw\bin
+psql -U gpadmin -p 15432 -h 127.0.0.1 -c "select version();" "dbname=postgres" || goto :error
+psql -U gpadmin -p 15432 -h 127.0.0.1 -c "select version();" "dbname=postgres sslmode=require" || goto :error
+start /B pipe_win10.exe
+start /B gpfdist.exe -d \\.\pipe\
+curl -H "X-GP-PROTO: 0" http://127.0.0.1:8080/public_test_0_pipe0 || goto :error
+cd gpload2
+python TEST_REMOTE.py || goto :error
+exit /b 0
+
+:error
+echo Failed with error #%errorlevel%.
+exit /b %errorlevel%
\ No newline at end of file

concourse/scripts/windows_remote_test.ps1

+Invoke-WebRequest -Uri https://aka.ms/vs/15/release/VC_redist.x64.exe -OutFile VC_redist.x64.exe
+Start-Process -FilePath "VC_redist.x64.exe" -ArgumentList "/passive" -Wait -Passthru
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+Invoke-WebRequest -Uri https://curl.haxx.se/windows/dl-7.64.1/curl-7.64.1-win64-mingw.zip -OutFile curl-7.64.1-win64-mingw.zip
+Expand-Archive -LiteralPath curl-7.64.1-win64-mingw.zip -DestinationPath "C:\Program Files\"
+move "C:\Program Files\curl-*-win64-mingw" "C:\Program Files\curl-win64-mingw"
+Start-Process msiexec.exe -Wait -ArgumentList '/I greenplum-clients-x86_64.msi /quiet'
+$env:PATH="C:\Program Files\Greenplum\greenplum-clients\bin;C:\Program Files\curl-win64-mingw\bin;" + $env:PATH

concourse/tasks/compile_gpdb_remote_windows.yml

@@ -12,4 +12,3 @@ params:
   REMOTE_PORT:
   REMOTE_USER:
   REMOTE_KEY:
-

concourse/tasks/rename_rc_artifacts.yml

@@ -13,6 +13,7 @@ inputs:
 - name: bin_gpdb_clients_centos6
 - name: bin_gpdb_clients_centos7
 - name: bin_gpdb_clients_ubuntu18.04
+- name: bin_gpdb_clients_windows
 
 outputs:
 - name: release_candidates
@@ -31,4 +32,8 @@ run:
     cp -v bin_gpdb_ubuntu18.04/bin_gpdb.tar.gz release_candidates/server-rc-${gpdb_semver}-ubuntu18.04_x86_64${RC_BUILD_TYPE_GCS}.tar.gz
     cp -v bin_gpdb_clients_centos6/bin_gpdb_clients.tar.gz release_candidates/clients-rc-${gpdb_semver}-rhel6_x86_64${RC_BUILD_TYPE_GCS}.tar.gz
     cp -v bin_gpdb_clients_centos7/bin_gpdb_clients.tar.gz release_candidates/clients-rc-${gpdb_semver}-rhel7_x86_64${RC_BUILD_TYPE_GCS}.tar.gz
-    cp -v bin_gpdb_clients_ubuntu18.04/bin_gpdb_clients.tar.gz release_candidates/clients-rc-${gpdb_semver}-ubuntu18.04_x86_64${RC_BUILD_TYPE_GCS}.tar.gz
\ No newline at end of file
+    cp -v bin_gpdb_clients_ubuntu18.04/bin_gpdb_clients.tar.gz release_candidates/clients-rc-${gpdb_semver}-ubuntu18.04_x86_64${RC_BUILD_TYPE_GCS}.tar.gz
+    pushd bin_gpdb_clients_windows
+        tar xzvf *.tar.gz
+    popd
+    cp bin_gpdb_clients_windows/greenplum-clients-x86_64.msi release_candidates/clients-rc-${gpdb_semver}-windows_x86_64.msi

concourse/tasks/windows_remote_test.yml

+platform: linux
+inputs:
+  - name: terraform_windows
+  - name: gpdb_src
+  - name: bin_gpdb
+  - name: bin_gpdb_clients_windows
+outputs:
+  - name: bin_gpdb_clients_windows_rc
+run:
+  path: gpdb_src/concourse/scripts/ic_gpdb_remote_windows.bash
+params:
+  REMOTE_PORT:
+  REMOTE_USER:
+  REMOTE_KEY:

gpAux/client/install/src/windows/CreatePackage.bat

 set GPDB_INSTALL_PATH=%1
 set VERSION=%2
+echo %VERSION% > %GPDB_INSTALL_PATH%\VERSION
 copy ..\..\..\..\..\NOTICE %GPDB_INSTALL_PATH%
 copy ..\..\..\..\..\LICENSE %GPDB_INSTALL_PATH%
 copy ..\..\..\scripts\greenplum_clients_path.bat %GPDB_INSTALL_PATH%
@@ -10,4 +11,4 @@ copy ..\..\..\..\..\gpMgmt\bin\gpload.bat %GPDB_INSTALL_PATH%\bin
 copy ..\..\..\..\..\gpMgmt\bin\ext\yaml\* %GPDB_INSTALL_PATH%\lib\python\yaml
 perl -p -e "s,__VERSION_PLACEHOLDER__,%VERSION%," greenplum-clients.wxs > greenplum-clients-%VERSION%.wxs
 candle.exe -nologo greenplum-clients-%VERSION%.wxs -out greenplum-clients-%VERSION%.wixobj -dSRCDIR=%GPDB_INSTALL_PATH% -dVERSION=%VERSION%
-light.exe -nologo -sval greenplum-clients-%VERSION%.wixobj -out greenplum-clients-%VERSION%-x86_64.msi
\ No newline at end of file
+light.exe -nologo -sval greenplum-clients-%VERSION%.wixobj -out greenplum-clients-x86_64.msi
\ No newline at end of file

gpAux/client/install/src/windows/greenplum-clients.wxs

@@ -1095,7 +1095,7 @@ If you want to review or change any of your installation settings, click Back. C
     <Directory Id="TARGETDIR" Name="SourceDir">
       <Directory Id="ProgramFiles64Folder" Name="PFiles">
         <Directory Id="gpdir" Name="Greenplum">
-          <Directory Id="BASEDIR" Name="greenplum-clients-$(var.VERSION)" ShortName="bizgres1">
+          <Directory Id="BASEDIR" Name="greenplum-clients" ShortName="bizgres1">
             <Component Id="TOPLEVELTextFiles" DiskId="1" Guid="CBED1D6A-228F-4AD2-AAD3-198B42323A22">
               <Environment Id="CLIENTS_ENV" Action="set" Permanent="no" System="yes" Part="all" Name="GPHOME_CLIENTS" Value="[BASEDIR]" />
               <Environment Id="PYTHON_ENV" Action="set" Permanent="no" System="yes" Part="last" Name="PYTHONPATH" Value="%GPHOME_CLIENTS%\lib\python" />
@@ -1103,6 +1103,7 @@ If you want to review or change any of your installation settings, click Back. C
               <File Id="greenplum_clients_path.bat" Name="greenplum_clients_path.bat" ShortName="BIZGRES1.BAT" Source="$(var.SRCDIR)\greenplum_clients_path.bat" />
               <File Id="LICENSE" Name="Pivotal License" ShortName="LICENSE" Source="$(var.SRCDIR)\LICENSE" />
               <File Id="NOTICE" Name="Thirdpary Notice" ShortName="NOTICE" Source="$(var.SRCDIR)\NOTICE" />
+              <File Id="VERSION" Name="GPDB Clients Version" ShortName="VERSION" Source="$(var.SRCDIR)\VERSION" />
             </Component>
             <Directory Id="TOPLEVELbin" Name="bin">
               <Component Id="TOPLEVELbin" DiskId="1" Guid="33133D23-0F1E-4A52-B679-D57D0A7DF727">

gpMgmt/bin/gpload_test/gpload2/TEST_REMOTE.py

@@ -10,6 +10,7 @@ import fileinput
 import platform
 import re
 import subprocess
+from shutil import copyfile
 from pygresql import pg
 
 """
@@ -40,9 +41,9 @@ d = mkpath('config')
 if not os.path.exists(d):
     os.mkdir(d)
 
-def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False'):
+def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file=os.path.join('data','external_file_01.txt'),table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False'):
 
-    f = open(mkpath('config/config_file'),'w')
+    f = open(mkpath(os.path.join('config','config_file')),'w')
     f.write("VERSION: 1.0.0.1")
     if database:
         f.write("\nDATABASE: "+database)
@@ -182,7 +183,10 @@ def psql_run(ifile = None, ofile = None, cmd = None,
     if ofile == '-':
         ofile = '2>&1'
     elif not ofile:
-        ofile = '> /dev/null 2>&1'
+        if (platform.system()) in ['Windows', 'Microsoft']:
+            ofile = '>  NUL'
+        else:
+            ofile = '> /dev/null 2>&1'
     else:
         ofile = '> %s 2>&1' % ofile
 
@@ -218,9 +222,9 @@ def changeExtFile( fname, ext = ".diff", outputPath = "" ):
     if len( outputPath ) == 0:
         return os.path.splitext( fname )[0] + ext
     else:
-        filename = fname.split( "/" )
+        filename = fname.split( os.sep )
         fname = os.path.splitext( filename[len( filename ) - 1] )[0]
-        return outputPath + "/" + fname + ext
+        return outputPath + os.sep + fname + ext
 
 def gpdbAnsFile(fname):
     ext = '.ans'
@@ -235,21 +239,23 @@ def isFileEqual( f1, f2, optionalFlags = "", outputPath = "", myinitfile = ""):
         raise Exception( 'Error: cannot find file %s' % f2 )
     dfile = diffFile( f1, outputPath = outputPath )
     # Gets the suitePath name to add init_file
-    suitePath = f1[0:f1.rindex( "/" )]
-    if os.path.exists(suitePath + "/init_file"):
+    suitePath = f1[0:f1.rindex( os.sep )]
+    global_init_file = os.path.join(LMYD, "global_init_file")
+    init_file = os.path.join(suitePath, "init_file")
+    if os.path.exists(os.path.join(suitePath, "init_file")):
         (ok, out) = run('gpdiff.pl -w ' + optionalFlags + \
-                              ' --gp_init_file=%s/global_init_file --gp_init_file=%s/init_file '
-                              '%s %s > %s 2>&1' % (LMYD, suitePath, f1, f2, dfile))
+                              ' --gp_init_file=%s --gp_init_file=%s '
+                              '%s %s > %s 2>&1' % (global_init_file, init_file, f1, f2, dfile))
 
     else:
         if os.path.exists(myinitfile):
             (ok, out) = run('gpdiff.pl -w ' + optionalFlags + \
-                                  ' -I NOTICE: -I HINT: -I CONTEXT: -I GP_IGNORE: --gp_init_file=%s/global_init_file --gp_init_file=%s '
-                                  '%s %s > %s 2>&1' % (LMYD, myinitfile, f1, f2, dfile))
+                                  ' -I NOTICE: -I HINT: -I CONTEXT: -I GP_IGNORE: --gp_init_file=%s --gp_init_file=%s '
+                                  '%s %s > %s 2>&1' % (global_init_file, myinitfile, f1, f2, dfile))
         else:
             (ok, out) = run( 'gpdiff.pl -w ' + optionalFlags + \
-                              ' -I NOTICE: -I HINT: -I CONTEXT: -I GP_IGNORE: --gp_init_file=%s/global_init_file '
-                              '%s %s > %s 2>&1' % ( LMYD, f1, f2, dfile ) )
+                              ' -I NOTICE: -I HINT: -I CONTEXT: -I GP_IGNORE: --gp_init_file=%s '
+                              '%s %s > %s 2>&1' % ( global_init_file, f1, f2, dfile ) )
 
 
     if ok:
@@ -266,9 +272,7 @@ def read_diff(ifile, outputPath):
         return diff.read()
 
 def copy_data(source='',target=''):
-    cmd = 'cp '+ mkpath('data/' + source) + ' ' + mkpath(target)
-    p = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
-    return p.communicate()
+    copyfile(os.path.join('data', source), target)
 
 def get_table_name():
     try:
@@ -349,13 +353,23 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
     def doTest(self, num, query=''):
         file = mkpath('query%d.diff' % num)
         if os.path.isfile(file):
-           run("rm -f" + " " + file)
-        f = open(mkpath('run_gpload.sh'), 'w')
-        f.write("gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n"+"gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n" )
+           os.remove(file)
+        ext = '.sh'
+
+        commands = "gpload -f "+mkpath(os.path.join('config','config_file'))+" -d reuse_gptest"+os.linesep+"gpload -f "+mkpath(os.path.join('config','config_file'))+ " -d reuse_gptest\n"
+        if (platform.system()) in ['Windows', 'Microsoft']:
+            ext = '.bat'
+            commands = "@ECHO OFF" + os.linesep+ " call gpload -f "+mkpath(os.path.join('config','config_file'))+ " -d reuse_gptest"+os.linesep+"call gpload -f "+mkpath(os.path.join('config','config_file'))+ " -d reuse_gptest" + os.linesep
+        f = open(mkpath('run_gpload' + ext), 'w')
+        f.write(commands)
         f.write(query)
         f.close()
 
         cmd = 'sh run_gpload.sh > query%d.out 2>&1' % num
+        if (platform.system()) in ['Windows', 'Microsoft']:
+            cmd = 'call run_gpload.bat > query%d.out 2>&1' % num
+
+
         run(cmd)
         self.check_result(file)
 
@@ -406,7 +420,7 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
         "7  gpload insert mode without reuse"
         runfile(mkpath('setup.sql'))
         write_config_file(mode='insert',reuse_flag='false')
-        self.doTest(7, "psql -d reuse_gptest -c 'select count(*) from texttable;'")
+        self.doTest(7, 'psql -d reuse_gptest -c "select count(*) from texttable;"')
 
     def test_08_gpload_reuse_table_update_mode_with_reuse(self):
         "8  gpload update mode with reuse"
@@ -419,7 +433,7 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
         "9  gpload update mode without reuse"
         copy_data('external_file_05.txt','data_file.txt')
         write_config_file(mode='update',reuse_flag='false',file='data_file.txt')
-        self.doTest(9, "psql -d reuse_gptest -c 'select count(*) from texttable;'\n"+"psql -d reuse_gptest -c 'select * from texttable where n2=222;'")
+        self.doTest(9, 'psql -d reuse_gptest -c "select count(*) from texttable;"\n'+'psql -d reuse_gptest -c "select * from texttable where n2=222;"')
 
     def test_10_gpload_reuse_table_merge_mode_with_reuse(self):
         "10  gpload merge mode with reuse "
@@ -456,8 +470,8 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
 
     def test_15_gpload_reuse_table_merge_mode_with_different_columns_order(self):
         "15 gpload merge mode with different columns' order "
-        copy_data('external_file_10.txt','data/data_file.tbl')
-        write_config_file('merge','true',file='data/data_file.tbl',columns_flag='1',mapping='1')
+        copy_data('external_file_10.txt',os.path.join('data','data_file.tbl'))
+        write_config_file('merge','true',file=os.path.join('data','data_file.tbl'),columns_flag='1',mapping='1')
         self.doTest(15)
 
     def test_16_gpload_formatOpts_quote(self):

src/bin/gpfdist/wintest/pipe_win10.cpp

+// gpload_win10.cpp : Defines the entry point for the console application.
+//
+
+#include <Windows.h>
+#include <stdio.h>
+#include <string>
+#define PIPEBUFF 4096	
+
+int main(int argc, char* argv[])
+{
+	std::string tmp = "\\\\.\\pipe\\public_test_0_pipe0";
+	HANDLE hPipe = CreateNamedPipe(
+		tmp.c_str(),             // pipe name 
+		PIPE_ACCESS_DUPLEX,       // read/write access 
+		PIPE_TYPE_MESSAGE |       // message type pipe 
+		PIPE_READMODE_MESSAGE |   // message-read mode 
+		PIPE_WAIT,                // blocking mode 
+		1,                        // max. instances  
+		0,                  // output buffer size 
+		0, // input buffer size
+		5000,                     // client time-out 
+		NULL);
+	if (hPipe == INVALID_HANDLE_VALUE)
+	{
+		printf("Failed to create pipe. Invalid pipe handle.\n");
+		return -1;
+	}
+	ConnectNamedPipe(hPipe, 0);
+	//Write data to file.
+	char DataBuffer[] = "1|one\n2|two\n3|three\n99|ninety nine";
+	printf("Data: [%s]\n", DataBuffer);
+	DWORD nBytesWritten = 0;
+	int nBytesToWrite = strlen(DataBuffer);
+	printf("Bytes to write = %d.\n", nBytesToWrite);
+	if (WriteFile(hPipe, DataBuffer, (DWORD)nBytesToWrite, &nBytesWritten, NULL) == 0)
+	{
+		printf("Failed to write to the pipe.\n");
+		return -1;
+	}
+	printf("Write to file succeded!\n");
+	DisconnectNamedPipe(hPipe);
+	CloseHandle(hPipe);
+	return 0;
+}

src/test/regress/gpdiff.pl

@@ -171,8 +171,8 @@ sub gpdiff_files
     }
 
     # replace temp file name references with actual file names
-    $outi =~ s/$newf1/$f1/gm;
-    $outi =~ s/$newf2/$f2/gm;
+    $outi =~ s/\Q$newf1\E/\Q$f1\E/gm;
+    $outi =~ s/\Q$newf2\E/\Q$f2\E/gm;
 
     print $outi;