Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Greenplum
Gpdb
提交
beabf1a2
G
Gpdb
项目概览
Greenplum
/
Gpdb
通知
7
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
G
Gpdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
beabf1a2
编写于
8月 30, 2016
作者:
P
Peifeng Qiu
提交者:
Adam Lee
10月 25, 2016
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
s3ext: update names, comments and delete dead codes
Signed-off-by:
N
Adam Lee
<
ali@pivotal.io
>
上级
dbea15a9
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
21 addition
and
25 deletion
+21
-25
gpAux/extensions/gps3ext/include/s3common.h
gpAux/extensions/gps3ext/include/s3common.h
+1
-3
gpAux/extensions/gps3ext/include/s3key_writer.h
gpAux/extensions/gps3ext/include/s3key_writer.h
+1
-1
gpAux/extensions/gps3ext/src/s3key_reader.cpp
gpAux/extensions/gps3ext/src/s3key_reader.cpp
+3
-4
gpAux/extensions/gps3ext/src/s3key_writer.cpp
gpAux/extensions/gps3ext/src/s3key_writer.cpp
+15
-16
gpAux/extensions/gps3ext/src/s3restful_service.cpp
gpAux/extensions/gps3ext/src/s3restful_service.cpp
+1
-1
未找到文件。
gpAux/extensions/gps3ext/include/s3common.h
浏览文件 @
beabf1a2
...
...
@@ -23,8 +23,6 @@ struct S3Credential {
string
secret
;
};
enum
Method
{
GET
,
PUT
,
POST
,
DELETE
,
HEAD
};
void
SignRequestV4
(
const
string
&
method
,
HTTPHeaders
*
h
,
const
string
&
orig_region
,
const
string
&
path
,
const
string
&
query
,
const
S3Credential
&
cred
);
...
...
@@ -32,4 +30,4 @@ string get_opt_s3(const string& options, const string& key);
string
truncate_options
(
const
string
&
url_with_options
);
#endif // __S3_COMMON_H__
#endif // __S3_COMMON_H__
\ No newline at end of file
gpAux/extensions/gps3ext/include/s3key_writer.h
浏览文件 @
beabf1a2
...
...
@@ -42,7 +42,7 @@ class S3KeyWriter : public Writer {
}
protected:
static
void
*
writerThread
(
void
*
p
);
static
void
*
UploadThreadFunc
(
void
*
p
);
void
flushBuffer
();
void
completeKeyWriting
();
...
...
gpAux/extensions/gps3ext/src/s3key_reader.cpp
浏览文件 @
beabf1a2
...
...
@@ -59,10 +59,9 @@ ChunkBuffer& ChunkBuffer::operator=(const ChunkBuffer& other) {
// that's why it checks if leftLen is larger than *or equal to* len below[1], provides a chance ret
// is 0, which is smaller than len. Otherwise, other functions won't know when to read next buffer.
uint64_t
ChunkBuffer
::
read
(
char
*
buf
,
uint64_t
len
)
{
// QueryCancelPending stops s3_import(), this check is not needed if
// s3_import() every time calls ChunkBuffer->Read() only once, otherwise(as we did in
// downstreamReader->read() for decompression feature before), first call sets buffer to
// ReadyToFill, second call hangs.
// GPDB abort signal stops s3_import(), this check is not needed if s3_import() every time calls
// ChunkBuffer->Read() only once, otherwise(as we did in downstreamReader->read() for
// decompression feature before), first call sets buffer to ReadyToFill, second call hangs.
CHECK_OR_DIE_MSG
(
!
S3QueryIsAbortInProgress
(),
"%s"
,
"ChunkBuffer reading is interrupted by user"
);
...
...
gpAux/extensions/gps3ext/src/s3key_writer.cpp
浏览文件 @
beabf1a2
...
...
@@ -77,33 +77,33 @@ struct ThreadParams {
uint64_t
currentNumber
;
};
void
*
S3KeyWriter
::
writerThread
(
void
*
p
)
{
ThreadParams
*
pa
ck
=
(
ThreadParams
*
)
p
;
S3KeyWriter
*
writer
=
pa
ck
->
keyWriter
;
void
*
S3KeyWriter
::
UploadThreadFunc
(
void
*
data
)
{
ThreadParams
*
pa
rams
=
(
ThreadParams
*
)
data
;
S3KeyWriter
*
writer
=
pa
rams
->
keyWriter
;
try
{
S3DEBUG
(
"Upload thread start: %p, part number: %"
PRIu64
", data size: %"
PRIu64
,
pthread_self
(),
pa
ck
->
currentNumber
,
pack
->
data
.
size
());
string
etag
=
writer
->
s3interface
->
uploadPartOfData
(
pack
->
data
,
writer
->
url
,
writer
->
region
,
writer
->
cred
,
pack
->
currentNumber
,
writer
->
uploadId
);
pthread_self
(),
pa
rams
->
currentNumber
,
params
->
data
.
size
());
string
etag
=
writer
->
s3interface
->
uploadPartOfData
(
params
->
data
,
writer
->
url
,
writer
->
region
,
writer
->
cred
,
params
->
currentNumber
,
writer
->
uploadId
);
// when unique_lock destructs it will automatically unlock the mutex.
UniqueLock
threadLock
(
&
writer
->
mutex
);
// etag is empty if the query is cancelled by user.
if
(
!
etag
.
empty
())
{
writer
->
etagList
[
pa
ck
->
currentNumber
]
=
etag
;
writer
->
etagList
[
pa
rams
->
currentNumber
]
=
etag
;
}
writer
->
activeThreads
--
;
pthread_cond_broadcast
(
&
writer
->
cv
);
S3DEBUG
(
"Upload part finish: %p, eTag: %s, part number: %"
PRIu64
,
pthread_self
(),
etag
.
c_str
(),
pa
ck
->
currentNumber
);
etag
.
c_str
(),
pa
rams
->
currentNumber
);
}
catch
(
std
::
exception
&
e
)
{
S3ERROR
(
"Upload thread error: %s"
,
e
.
what
());
}
delete
pa
ck
;
delete
pa
rams
;
return
NULL
;
}
...
...
@@ -114,20 +114,19 @@ void S3KeyWriter::flushBuffer() {
pthread_cond_wait
(
&
this
->
cv
,
&
this
->
mutex
);
}
// Most time query is canceled during uploadPartOfData. This is the first chance to cancel
// and clean up upload. Otherwise GPDB will call with LAST_CALL but QueryCancelPending is
// set to false, and we can't detect query cancel signal in S3KeyWriter::close().
// Most time query is canceled during uploadPartOfData(). This is the first chance to cancel
// and clean up upload.
this
->
checkQueryCancelSignal
();
this
->
activeThreads
++
;
pthread_t
t
hread
;
pthread_t
writerT
hread
;
ThreadParams
*
params
=
new
ThreadParams
();
params
->
keyWriter
=
this
;
params
->
data
.
swap
(
this
->
buffer
);
params
->
currentNumber
=
++
this
->
partNumber
;
pthread_create
(
&
thread
,
NULL
,
writerThread
,
params
);
threadList
.
emplace_back
(
t
hread
);
pthread_create
(
&
writerThread
,
NULL
,
UploadThreadFunc
,
params
);
threadList
.
emplace_back
(
writerT
hread
);
this
->
buffer
.
clear
();
}
...
...
gpAux/extensions/gps3ext/src/s3restful_service.cpp
浏览文件 @
beabf1a2
...
...
@@ -40,7 +40,7 @@ size_t RESTfulServiceWriteFuncCallback(char *ptr, size_t size, size_t nmemb, voi
}
// cURL's write function callback, only used by DELETE request when query is canceled.
// It shouldn't be interrupted
by QueryCancelPending
.
// It shouldn't be interrupted.
size_t
RESTfulServiceAbortFuncCallback
(
char
*
ptr
,
size_t
size
,
size_t
nmemb
,
void
*
userp
)
{
size_t
realsize
=
size
*
nmemb
;
Response
*
resp
=
(
Response
*
)
userp
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录