Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wux_labs
Tensorflow
提交
e376278d
T
Tensorflow
项目概览
wux_labs
/
Tensorflow
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Tensorflow
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e376278d
编写于
10月 24, 2022
作者:
A
A. Unique TensorFlower
提交者:
Gagik Amirkhanyan
10月 29, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Make GCS filesystem/metadata lookup retries configurable
PiperOrigin-RevId: 483432073
上级
829f9ebd
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
50 addition
and
4 deletion
+50
-4
tensorflow/core/platform/cloud/gcs_file_system.cc
tensorflow/core/platform/cloud/gcs_file_system.cc
+49
-1
tensorflow/core/platform/cloud/gcs_file_system.h
tensorflow/core/platform/cloud/gcs_file_system.h
+1
-3
未找到文件。
tensorflow/core/platform/cloud/gcs_file_system.cc
浏览文件 @
e376278d
...
@@ -131,6 +131,16 @@ constexpr char kThrottleBucket[] = "GCS_THROTTLE_BUCKET_SIZE";
...
@@ -131,6 +131,16 @@ constexpr char kThrottleBucket[] = "GCS_THROTTLE_BUCKET_SIZE";
constexpr
char
kTokensPerRequest
[]
=
"GCS_TOKENS_PER_REQUEST"
;
constexpr
char
kTokensPerRequest
[]
=
"GCS_TOKENS_PER_REQUEST"
;
// The environment variable to configure the initial tokens (format: <int64_t>)
// The environment variable to configure the initial tokens (format: <int64_t>)
constexpr
char
kInitialTokens
[]
=
"GCS_INITIAL_TOKENS"
;
constexpr
char
kInitialTokens
[]
=
"GCS_INITIAL_TOKENS"
;
// The environment variable for GCS retry init_delay_time_us (format: <int64_t>)
constexpr
char
kRetryConfigInitialDelayTimeUs
[]
=
"GCS_RETRY_CONFIG_INIT_DELAY_TIME_US"
;
// The environment variable for GCS retry config max_delay_time_us (format:
// <int64_t>)
constexpr
char
kRetryConfigMaxDelayTimeUs
[]
=
"GCS_RETRY_CONFIG_MAX_DELAY_TIME_US"
;
// The environment variable that controls the number of retries in GCS
// exponential retries (format: <int32_t>)
constexpr
char
kRetryConfigMaxRetries
[]
=
"GCS_RETRY_CONFIG_MAX_RETRIES"
;
// The environment variable to customize which GCS bucket locations are allowed,
// The environment variable to customize which GCS bucket locations are allowed,
// if the list is empty defaults to using the region of the zone (format, comma
// if the list is empty defaults to using the region of the zone (format, comma
...
@@ -265,6 +275,38 @@ Status GetBoolValue(const Json::Value& parent, const char* name, bool* result) {
...
@@ -265,6 +275,38 @@ Status GetBoolValue(const Json::Value& parent, const char* name, bool* result) {
return
OkStatus
();
return
OkStatus
();
}
}
/// Get GCS Retry Config by applying user overrides through env if any.
RetryConfig
GetGcsRetryConfig
()
{
RetryConfig
retryConfig
(
/* init_delay_time_us = */
1000
*
1000
,
/* max_delay_time_us = */
32
*
1000
*
1000
,
/* max_retries = */
10
);
// Apply the overrides for Retry configs.
uint64
init_delay_time_us
;
if
(
GetEnvVar
(
kRetryConfigInitialDelayTimeUs
,
strings
::
safe_strtou64
,
&
init_delay_time_us
))
{
retryConfig
.
init_delay_time_us
=
init_delay_time_us
;
}
uint64
max_delay_time_us
;
if
(
GetEnvVar
(
kRetryConfigMaxDelayTimeUs
,
strings
::
safe_strtou64
,
&
max_delay_time_us
))
{
retryConfig
.
max_delay_time_us
=
max_delay_time_us
;
}
uint32
max_retries
;
if
(
GetEnvVar
(
kRetryConfigMaxRetries
,
strings
::
safe_strtou32
,
&
max_retries
))
{
retryConfig
.
max_retries
=
max_retries
;
}
VLOG
(
1
)
<<
"GCS RetryConfig: "
<<
"init_delay_time_us = "
<<
retryConfig
.
init_delay_time_us
<<
" ; "
<<
"max_delay_time_us = "
<<
retryConfig
.
max_delay_time_us
<<
" ; "
<<
"max_retries = "
<<
retryConfig
.
max_retries
;
return
retryConfig
;
}
/// A GCS-based implementation of a random access file with an LRU block cache.
/// A GCS-based implementation of a random access file with an LRU block cache.
class
GcsRandomAccessFile
:
public
RandomAccessFile
{
class
GcsRandomAccessFile
:
public
RandomAccessFile
{
public:
public:
...
@@ -727,7 +769,7 @@ class GcsWritableFile : public WritableFile {
...
@@ -727,7 +769,7 @@ class GcsWritableFile : public WritableFile {
GcsFileSystem
::
TimeoutConfig
*
timeouts_
;
GcsFileSystem
::
TimeoutConfig
*
timeouts_
;
std
::
function
<
void
()
>
file_cache_erase_
;
std
::
function
<
void
()
>
file_cache_erase_
;
bool
sync_needed_
;
// whether there is buffered data that needs to be synced
bool
sync_needed_
;
// whether there is buffered data that needs to be synced
RetryConfig
retry_config_
;
RetryConfig
retry_config_
=
GetGcsRetryConfig
()
;
bool
compose_append_
;
bool
compose_append_
;
uint64
start_offset_
;
uint64
start_offset_
;
// Callbacks to the file system used to upload object into GCS.
// Callbacks to the file system used to upload object into GCS.
...
@@ -929,6 +971,8 @@ GcsFileSystem::GcsFileSystem(bool make_default_cache) {
...
@@ -929,6 +971,8 @@ GcsFileSystem::GcsFileSystem(bool make_default_cache) {
}
else
{
}
else
{
compose_append_
=
false
;
compose_append_
=
false
;
}
}
retry_config_
=
GetGcsRetryConfig
();
}
}
GcsFileSystem
::
GcsFileSystem
(
GcsFileSystem
::
GcsFileSystem
(
...
@@ -2129,6 +2173,10 @@ Status GcsFileSystem::CreateHttpRequest(std::unique_ptr<HttpRequest>* request) {
...
@@ -2129,6 +2173,10 @@ Status GcsFileSystem::CreateHttpRequest(std::unique_ptr<HttpRequest>* request) {
return
OkStatus
();
return
OkStatus
();
}
}
RetryingGcsFileSystem
::
RetryingGcsFileSystem
()
:
RetryingFileSystem
(
std
::
make_unique
<
GcsFileSystem
>
(),
RetryConfig
(
GetGcsRetryConfig
()))
{}
}
// namespace tensorflow
}
// namespace tensorflow
// The TPU_GCS_FS option sets a TPU-on-GCS optimized file system that allows
// The TPU_GCS_FS option sets a TPU-on-GCS optimized file system that allows
...
...
tensorflow/core/platform/cloud/gcs_file_system.h
浏览文件 @
e376278d
...
@@ -447,9 +447,7 @@ class GcsFileSystem : public FileSystem {
...
@@ -447,9 +447,7 @@ class GcsFileSystem : public FileSystem {
/// Google Cloud Storage implementation of a file system with retry on failures.
/// Google Cloud Storage implementation of a file system with retry on failures.
class
RetryingGcsFileSystem
:
public
RetryingFileSystem
<
GcsFileSystem
>
{
class
RetryingGcsFileSystem
:
public
RetryingFileSystem
<
GcsFileSystem
>
{
public:
public:
RetryingGcsFileSystem
()
RetryingGcsFileSystem
();
:
RetryingFileSystem
(
std
::
unique_ptr
<
GcsFileSystem
>
(
new
GcsFileSystem
),
RetryConfig
(
100000
/* init_delay_time_us */
))
{}
};
};
}
// namespace tensorflow
}
// namespace tensorflow
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录