Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
曾经的那一瞬间
Models
提交
20e2cb97
M
Models
项目概览
曾经的那一瞬间
/
Models
大约 1 年 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
Models
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
20e2cb97
编写于
8月 27, 2020
作者:
R
Ruoxin Sang
提交者:
A. Unique TensorFlower
8月 27, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Internal change
PiperOrigin-RevId: 328789639
上级
454e12b8
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
57 addition
and
4 deletion
+57
-4
official/common/flags.py
official/common/flags.py
+3
-0
official/core/input_reader.py
official/core/input_reader.py
+30
-3
official/core/train_utils.py
official/core/train_utils.py
+9
-1
official/modeling/hyperparams/config_definitions.py
official/modeling/hyperparams/config_definitions.py
+15
-0
未找到文件。
official/common/flags.py
浏览文件 @
20e2cb97
...
...
@@ -75,3 +75,6 @@ def define_flags():
help
=
'The Cloud TPU to use for training. This should be either the name '
'used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 '
'url.'
)
flags
.
DEFINE_string
(
'tf_data_service'
,
default
=
None
,
help
=
'The tf.data service address'
)
official/core/input_reader.py
浏览文件 @
20e2cb97
...
...
@@ -100,6 +100,7 @@ class InputReader:
self
.
_cache
=
params
.
cache
self
.
_cycle_length
=
params
.
cycle_length
self
.
_block_length
=
params
.
block_length
self
.
_deterministic
=
params
.
deterministic
self
.
_sharding
=
params
.
sharding
self
.
_examples_consume
=
params
.
examples_consume
self
.
_tfds_split
=
params
.
tfds_split
...
...
@@ -114,6 +115,11 @@ class InputReader:
self
.
_postprocess_fn
=
postprocess_fn
self
.
_seed
=
_get_random_integer
()
self
.
_enable_tf_data_service
=
(
params
.
enable_tf_data_service
and
params
.
tf_data_service_address
)
self
.
_tf_data_service_address
=
params
.
tf_data_service_address
self
.
_tf_data_service_job_name
=
params
.
tf_data_service_job_name
def
_read_sharded_files
(
self
,
input_context
:
Optional
[
tf
.
distribute
.
InputContext
]
=
None
):
...
...
@@ -134,8 +140,11 @@ class InputReader:
seed
=
self
.
_seed
,
reshuffle_each_iteration
=
True
)
# Do not enable sharding if tf.data service is enabled, as sharding will be
# handled inside tf.data service.
if
self
.
_sharding
and
input_context
and
(
input_context
.
num_input_pipelines
>
1
):
input_context
.
num_input_pipelines
>
1
and
not
self
.
_enable_tf_data_service
):
dataset
=
dataset
.
shard
(
input_context
.
num_input_pipelines
,
input_context
.
input_pipeline_id
)
if
self
.
_is_training
:
...
...
@@ -145,7 +154,8 @@ class InputReader:
map_func
=
self
.
_dataset_fn
,
cycle_length
=
self
.
_cycle_length
,
block_length
=
self
.
_block_length
,
num_parallel_calls
=
tf
.
data
.
experimental
.
AUTOTUNE
)
num_parallel_calls
=
tf
.
data
.
experimental
.
AUTOTUNE
,
deterministic
=
self
.
_deterministic
)
return
dataset
def
_read_single_file
(
...
...
@@ -161,8 +171,11 @@ class InputReader:
options
.
experimental_distribute
.
auto_shard_policy
=
(
tf
.
data
.
experimental
.
AutoShardPolicy
.
OFF
)
dataset
=
dataset
.
with_options
(
options
)
# Do not enable sharding if tf.data service is enabled, as sharding will be
# handled inside tf.data service.
if
self
.
_sharding
and
input_context
and
(
input_context
.
num_input_pipelines
>
1
):
input_context
.
num_input_pipelines
>
1
and
not
self
.
_enable_tf_data_service
):
dataset
=
dataset
.
shard
(
input_context
.
num_input_pipelines
,
input_context
.
input_pipeline_id
)
if
self
.
_is_training
:
...
...
@@ -243,4 +256,18 @@ class InputReader:
per_replica_batch_size
,
drop_remainder
=
self
.
_drop_remainder
)
dataset
=
maybe_map_fn
(
dataset
,
self
.
_postprocess_fn
)
if
self
.
_enable_tf_data_service
:
dataset
=
dataset
.
apply
(
tf
.
data
.
experimental
.
service
.
distribute
(
processing_mode
=
'parallel_epochs'
,
service
=
self
.
_tf_data_service_address
,
job_name
=
self
.
_tf_data_service_job_name
))
dataset
=
dataset
.
prefetch
(
buffer_size
=
tf
.
data
.
experimental
.
AUTOTUNE
)
if
self
.
_deterministic
is
not
None
:
options
=
tf
.
data
.
Options
()
options
.
experimental_deterministic
=
self
.
_deterministic
dataset
=
dataset
.
with_options
(
options
)
return
dataset
.
prefetch
(
tf
.
data
.
experimental
.
AUTOTUNE
)
official/core/train_utils.py
浏览文件 @
20e2cb97
...
...
@@ -60,10 +60,18 @@ def parse_configuration(flags_obj):
params
=
hyperparams
.
override_params_dict
(
params
,
config_file
,
is_strict
=
True
)
# 3. Override the TPU address.
# 3. Override the TPU address
and tf.data service address
.
params
.
override
({
'runtime'
:
{
'tpu'
:
flags_obj
.
tpu
,
},
'task'
:
{
'train_data'
:
{
'tf_data_service_address'
:
flags_obj
.
tf_data_service
,
},
'validation_data'
:
{
'tf_data_service_address'
:
flags_obj
.
tf_data_service
,
}
}
})
...
...
official/modeling/hyperparams/config_definitions.py
浏览文件 @
20e2cb97
...
...
@@ -48,11 +48,22 @@ class DataConfig(base_config.Config):
interleaving files.
block_length: The number of consecutive elements to produce from each input
element before cycling to another input element when interleaving files.
deterministic: A boolean controlling whether determinism should be enforced.
sharding: Whether sharding is used in the input pipeline.
examples_consume: An `integer` specifying the number of examples it will
produce. If positive, it only takes this number of examples and raises
tf.error.OutOfRangeError after that. Default is -1, meaning it will
exhaust all the examples in the dataset.
enable_tf_data_service: A boolean indicating whether to enable tf.data
service for the input pipeline.
tf_data_service_address: The URI of a tf.data service to offload
preprocessing onto during training. The URI should be in the format
"protocol://address", e.g. "grpc://tf-data-service:5050". It can be
overridden by `FLAGS.tf_data_service` flag in the binary.
tf_data_service_job_name: The name of the tf.data service job. This
argument makes it possible for multiple datasets to share the same job.
The default behavior is that the dataset creates anonymous, exclusively
owned jobs.
tfds_data_dir: A str specifying the directory to read/write TFDS data.
tfds_download: A bool to indicate whether to download data using TFDS.
tfds_as_supervised: A bool. When loading dataset from TFDS, if True, the
...
...
@@ -74,8 +85,12 @@ class DataConfig(base_config.Config):
cache
:
bool
=
False
cycle_length
:
int
=
8
block_length
:
int
=
1
deterministic
:
Optional
[
bool
]
=
None
sharding
:
bool
=
True
examples_consume
:
int
=
-
1
enable_tf_data_service
:
bool
=
False
tf_data_service_address
:
Optional
[
str
]
=
None
tf_data_service_job_name
:
Optional
[
str
]
=
None
tfds_data_dir
:
str
=
""
tfds_download
:
bool
=
False
tfds_as_supervised
:
bool
=
False
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录