Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Greenplum
Gpdb
提交
d48e624b
G
Gpdb
项目概览
Greenplum
/
Gpdb
通知
7
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
G
Gpdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d48e624b
编写于
8月 15, 2016
作者:
P
Peifeng Qiu
提交者:
Adam Lee
8月 29, 2016
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
s3ext: merge ReaderParams and WriterParams
上级
73e85698
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
53 addition
and
130 deletion
+53
-130
gpAux/extensions/gps3ext/include/reader.h
gpAux/extensions/gps3ext/include/reader.h
+1
-1
gpAux/extensions/gps3ext/include/s3params.h
gpAux/extensions/gps3ext/include/s3params.h
+35
-29
gpAux/extensions/gps3ext/include/writer.h
gpAux/extensions/gps3ext/include/writer.h
+1
-1
gpAux/extensions/gps3ext/include/writer_params.h
gpAux/extensions/gps3ext/include/writer_params.h
+0
-83
gpAux/extensions/gps3ext/src/gpreader.cpp
gpAux/extensions/gps3ext/src/gpreader.cpp
+1
-1
gpAux/extensions/gps3ext/src/gpwriter.cpp
gpAux/extensions/gps3ext/src/gpwriter.cpp
+2
-2
gpAux/extensions/gps3ext/src/s3bucket_reader.cpp
gpAux/extensions/gps3ext/src/s3bucket_reader.cpp
+2
-2
gpAux/extensions/gps3ext/test/s3bucket_reader_test.cpp
gpAux/extensions/gps3ext/test/s3bucket_reader_test.cpp
+9
-9
gpAux/extensions/gps3ext/test/s3key_reader_test.cpp
gpAux/extensions/gps3ext/test/s3key_reader_test.cpp
+1
-1
gpAux/extensions/gps3ext/test/s3key_writer_test.cpp
gpAux/extensions/gps3ext/test/s3key_writer_test.cpp
+1
-1
未找到文件。
gpAux/extensions/gps3ext/include/reader.h
浏览文件 @
d48e624b
#ifndef __GP_EXT_READER_H__
#define __GP_EXT_READER_H__
#include "
reader_
params.h"
#include "
s3
params.h"
class
Reader
{
public:
...
...
gpAux/extensions/gps3ext/include/
reader_
params.h
→
gpAux/extensions/gps3ext/include/
s3
params.h
浏览文件 @
d48e624b
#ifndef
INCLUDE_READER_PARAMS_H
_
#define
INCLUDE_READER_PARAMS_H
_
#ifndef
__S3_PARAMS_H_
_
#define
__S3_PARAMS_H_
_
#include <string>
...
...
@@ -7,11 +7,11 @@
using
std
::
string
;
class
Reader
Params
{
class
S3
Params
{
public:
Reader
Params
()
:
keySize
(
0
),
chunkSize
(
0
),
numOfChunks
(
0
),
segId
(
0
),
segNum
(
1
)
{
S3
Params
()
:
keySize
(
0
),
chunkSize
(
0
),
numOfChunks
(
0
),
segId
(
0
),
segNum
(
1
)
{
}
virtual
~
Reader
Params
()
{
virtual
~
S3
Params
()
{
}
uint64_t
getChunkSize
()
const
{
...
...
@@ -30,14 +30,6 @@ class ReaderParams {
this
->
cred
=
cred
;
}
const
string
&
getKeyUrl
()
const
{
return
keyUrl
;
}
void
setKeyUrl
(
const
string
&
keyUrl
)
{
this
->
keyUrl
=
keyUrl
;
}
const
string
&
getRegion
()
const
{
return
region
;
}
...
...
@@ -46,20 +38,12 @@ class ReaderParams {
this
->
region
=
region
;
}
uint64_t
getKeySize
()
const
{
return
keySize
;
}
void
setKeySize
(
uint64_t
size
)
{
this
->
keySize
=
size
;
}
const
string
&
getUrlToLoad
()
const
{
return
urlToLoad
;
const
string
&
getKeyUrl
()
const
{
return
keyUrl
;
}
void
set
UrlToLoad
(
const
string
&
url
)
{
this
->
urlToLoad
=
url
;
void
set
KeyUrl
(
const
string
&
url
)
{
this
->
keyUrl
=
url
;
}
uint64_t
getSegId
()
const
{
...
...
@@ -86,11 +70,29 @@ class ReaderParams {
this
->
numOfChunks
=
numOfChunks
;
}
uint64_t
getKeySize
()
const
{
return
keySize
;
}
void
setKeySize
(
uint64_t
size
)
{
this
->
keySize
=
size
;
}
const
string
&
getBaseUrl
()
const
{
return
baseUrl
;
}
void
setBaseUrl
(
const
string
&
url
)
{
this
->
baseUrl
=
url
;
}
private:
string
urlToLoad
;
// original url to read/write.
string
keyUrl
;
// key url in s3 bucket.
string
baseUrl
;
// original url to read/write.
string
keyUrl
;
uint64_t
keySize
;
// key/file size.
string
region
;
uint64_t
keySize
;
// key/file size.
uint64_t
chunkSize
;
// chunk size
uint64_t
numOfChunks
;
// number of chunks(threads).
S3Credential
cred
;
...
...
@@ -98,4 +100,8 @@ class ReaderParams {
uint64_t
segNum
;
};
#endif
/* INCLUDE_READER_PARAMS_H_ */
class
WriterParams
:
public
S3Params
{};
class
ReaderParams
:
public
S3Params
{};
#endif
\ No newline at end of file
gpAux/extensions/gps3ext/include/writer.h
浏览文件 @
d48e624b
#ifndef __GP_EXT_WRITER_H__
#define __GP_EXT_WRITER_H__
#include "
writer_
params.h"
#include "
s3
params.h"
class
Writer
{
public:
...
...
gpAux/extensions/gps3ext/include/writer_params.h
已删除
100644 → 0
浏览文件 @
73e85698
#ifndef INCLUDE_WRITER_PARAMS_H_
#define INCLUDE_WRITER_PARAMS_H_
#include <string>
#include "s3common.h"
using
std
::
string
;
class
WriterParams
{
public:
WriterParams
()
:
chunkSize
(
0
),
numOfChunks
(
0
),
segId
(
0
),
segNum
(
1
)
{
}
virtual
~
WriterParams
()
{
}
uint64_t
getChunkSize
()
const
{
return
chunkSize
;
}
void
setChunkSize
(
uint64_t
chunkSize
)
{
this
->
chunkSize
=
chunkSize
;
}
const
S3Credential
&
getCred
()
const
{
return
cred
;
}
void
setCred
(
const
S3Credential
&
cred
)
{
this
->
cred
=
cred
;
}
const
string
&
getRegion
()
const
{
return
region
;
}
void
setRegion
(
const
string
&
region
)
{
this
->
region
=
region
;
}
const
string
&
getKeyUrl
()
const
{
return
keyUrl
;
}
void
setKeyUrl
(
const
string
&
url
)
{
this
->
keyUrl
=
url
;
}
uint64_t
getSegId
()
const
{
return
segId
;
}
void
setSegId
(
uint64_t
segId
)
{
this
->
segId
=
segId
;
}
uint64_t
getSegNum
()
const
{
return
segNum
;
}
void
setSegNum
(
uint64_t
segNum
)
{
this
->
segNum
=
segNum
;
}
uint64_t
getNumOfChunks
()
const
{
return
numOfChunks
;
}
void
setNumOfChunks
(
uint64_t
numOfChunks
)
{
this
->
numOfChunks
=
numOfChunks
;
}
private:
string
keyUrl
;
string
region
;
uint64_t
chunkSize
;
// chunk size
uint64_t
numOfChunks
;
// number of chunks(threads).
S3Credential
cred
;
uint64_t
segId
;
uint64_t
segNum
;
};
#endif
/* INCLUDE_WRITER_PARAMS_H_ */
gpAux/extensions/gps3ext/src/gpreader.cpp
浏览文件 @
d48e624b
...
...
@@ -69,7 +69,7 @@ GPReader::GPReader(const string& url) {
}
void
GPReader
::
constructReaderParams
(
const
string
&
url
)
{
this
->
params
.
set
UrlToLoad
(
url
);
this
->
params
.
set
BaseUrl
(
url
);
this
->
params
.
setSegId
(
s3ext_segid
);
this
->
params
.
setSegNum
(
s3ext_segnum
);
this
->
params
.
setNumOfChunks
(
s3ext_threadnum
);
...
...
gpAux/extensions/gps3ext/src/gpwriter.cpp
浏览文件 @
d48e624b
...
...
@@ -24,7 +24,7 @@ GPWriter::GPWriter(const string& url) {
}
void
GPWriter
::
constructWriterParams
(
const
string
&
url
)
{
this
->
params
.
set
Key
Url
(
url
);
this
->
params
.
set
Base
Url
(
url
);
this
->
params
.
setSegId
(
s3ext_segid
);
this
->
params
.
setSegNum
(
s3ext_segnum
);
this
->
params
.
setNumOfChunks
(
s3ext_threadnum
);
...
...
@@ -38,7 +38,7 @@ void GPWriter::constructWriterParams(const string& url) {
void
GPWriter
::
open
(
const
WriterParams
&
params
)
{
this
->
s3service
.
setRESTfulService
(
this
->
restfulServicePtr
);
this
->
params
.
setKeyUrl
(
this
->
genUniqueKeyName
(
this
->
params
.
get
Key
Url
()));
this
->
params
.
setKeyUrl
(
this
->
genUniqueKeyName
(
this
->
params
.
get
Base
Url
()));
this
->
keyWriter
.
setS3interface
(
&
this
->
s3service
);
this
->
keyWriter
.
open
(
this
->
params
);
}
...
...
gpAux/extensions/gps3ext/src/s3bucket_reader.cpp
浏览文件 @
d48e624b
...
...
@@ -5,12 +5,12 @@
#include <inttypes.h>
#include "reader.h"
#include "reader_params.h"
#include "s3bucket_reader.h"
#include "s3common.h"
#include "s3conf.h"
#include "s3log.h"
#include "s3macros.h"
#include "s3params.h"
#include "s3utils.h"
using
std
::
string
;
...
...
@@ -36,7 +36,7 @@ S3BucketReader::~S3BucketReader() {
}
void
S3BucketReader
::
open
(
const
ReaderParams
&
params
)
{
this
->
url
=
params
.
get
UrlToLoad
();
this
->
url
=
params
.
get
BaseUrl
();
this
->
segId
=
params
.
getSegId
();
this
->
segNum
=
params
.
getSegNum
();
this
->
cred
=
params
.
getCred
();
...
...
gpAux/extensions/gps3ext/test/s3bucket_reader_test.cpp
浏览文件 @
d48e624b
...
...
@@ -2,8 +2,8 @@
#include "gtest/gtest.h"
#include "mock_classes.h"
#include "reader_params.h"
#include "s3bucket_reader.cpp"
#include "s3params.h"
using
::
testing
::
AtLeast
;
using
::
testing
::
Return
;
...
...
@@ -43,7 +43,7 @@ class S3BucketReaderTest : public testing::Test {
TEST_F
(
S3BucketReaderTest
,
OpenInvalidURL
)
{
string
url
=
"https://s3-us-east-2.amazon.com/s3test.pivotal.io/whatever"
;
params
.
set
UrlToLoad
(
url
);
params
.
set
BaseUrl
(
url
);
EXPECT_THROW
(
bucketReader
->
open
(
params
),
std
::
runtime_error
);
}
...
...
@@ -53,7 +53,7 @@ TEST_F(S3BucketReaderTest, OpenURL) {
.
WillOnce
(
Return
(
ListBucketResult
()));
string
url
=
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
;
params
.
set
UrlToLoad
(
url
);
params
.
set
BaseUrl
(
url
);
EXPECT_NO_THROW
(
bucketReader
->
open
(
params
));
}
...
...
@@ -62,7 +62,7 @@ TEST_F(S3BucketReaderTest, OpenThrowExceptionWhenS3InterfaceIsNULL) {
bucketReader
->
setS3interface
(
NULL
);
string
url
=
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
;
params
.
set
UrlToLoad
(
url
);
params
.
set
BaseUrl
(
url
);
EXPECT_THROW
(
bucketReader
->
open
(
params
),
std
::
runtime_error
);
}
...
...
@@ -145,7 +145,7 @@ TEST_F(S3BucketReaderTest, ReaderReturnZeroForEmptyBucket) {
.
Times
(
1
)
.
WillOnce
(
Return
(
ListBucketResult
()));
params
.
set
UrlToLoad
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
params
.
set
BaseUrl
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
bucketReader
->
open
(
params
);
bucketReader
->
setUpstreamReader
(
&
s3reader
);
EXPECT_EQ
(
0
,
bucketReader
->
read
(
buf
,
sizeof
(
buf
)));
...
...
@@ -168,7 +168,7 @@ TEST_F(S3BucketReaderTest, ReadBucketWithSingleFile) {
params
.
setSegId
(
0
);
params
.
setSegNum
(
1
);
params
.
set
UrlToLoad
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
params
.
set
BaseUrl
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
bucketReader
->
open
(
params
);
bucketReader
->
setUpstreamReader
(
&
s3reader
);
...
...
@@ -195,7 +195,7 @@ TEST_F(S3BucketReaderTest, ReadBuckeWithOneEmptyFileOneNonEmptyFile) {
params
.
setSegId
(
0
);
params
.
setSegNum
(
1
);
params
.
set
UrlToLoad
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
params
.
set
BaseUrl
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
bucketReader
->
open
(
params
);
bucketReader
->
setUpstreamReader
(
&
s3reader
);
...
...
@@ -211,7 +211,7 @@ TEST_F(S3BucketReaderTest, ReaderShouldSkipIfFileIsNotForThisSegment) {
params
.
setSegId
(
10
);
params
.
setSegNum
(
16
);
params
.
set
UrlToLoad
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
params
.
set
BaseUrl
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
bucketReader
->
open
(
params
);
bucketReader
->
setUpstreamReader
(
&
s3reader
);
...
...
@@ -234,7 +234,7 @@ TEST_F(S3BucketReaderTest, UpstreamReaderThrowException) {
params
.
setSegId
(
0
);
params
.
setSegNum
(
1
);
params
.
set
UrlToLoad
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
params
.
set
BaseUrl
(
"https://s3-us-east-2.amazonaws.com/s3test.pivotal.io/whatever"
);
bucketReader
->
open
(
params
);
bucketReader
->
setUpstreamReader
(
&
s3reader
);
...
...
gpAux/extensions/gps3ext/test/s3key_reader_test.cpp
浏览文件 @
d48e624b
...
...
@@ -2,8 +2,8 @@
#include "gtest/gtest.h"
#include "mock_classes.h"
#include "reader_params.h"
#include "s3key_reader.cpp"
#include "s3params.h"
using
::
testing
::
AtLeast
;
using
::
testing
::
AtMost
;
...
...
gpAux/extensions/gps3ext/test/s3key_writer_test.cpp
浏览文件 @
d48e624b
...
...
@@ -3,7 +3,7 @@
#include "mock_classes.h"
#include "s3key_writer.cpp"
#include "
writer_
params.h"
#include "
s3
params.h"
using
::
testing
::
AtLeast
;
using
::
testing
::
AtMost
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录