Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Oneflow-Inc
oneflow
提交
f5e8efae
O
oneflow
项目概览
Oneflow-Inc
/
oneflow
上一次同步 接近 3 年
通知
13
Star
2733
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
O
oneflow
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f5e8efae
编写于
12月 29, 2017
作者:
W
willzhang4a58
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
basic data loader
上级
c6a7a351
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
54 addition
and
28 deletion
+54
-28
oneflow/core/comm_network/rdma/rdma_memory.h
oneflow/core/comm_network/rdma/rdma_memory.h
+3
-3
oneflow/core/graph/backward_compute_task_node.h
oneflow/core/graph/backward_compute_task_node.h
+1
-1
oneflow/core/graph/forward_compute_task_node.h
oneflow/core/graph/forward_compute_task_node.h
+1
-1
oneflow/core/kernel/basic_data_loader_kernel.cpp
oneflow/core/kernel/basic_data_loader_kernel.cpp
+6
-6
oneflow/core/kernel/basic_data_loader_kernel.h
oneflow/core/kernel/basic_data_loader_kernel.h
+26
-0
oneflow/core/operator/basic_data_loader_op.cpp
oneflow/core/operator/basic_data_loader_op.cpp
+8
-8
oneflow/core/operator/basic_data_loader_op.h
oneflow/core/operator/basic_data_loader_op.h
+7
-7
oneflow/core/operator/op_conf.proto
oneflow/core/operator/op_conf.proto
+2
-2
未找到文件。
oneflow/core/comm_network/rdma/rdma_memory.h
浏览文件 @
f5e8efae
#ifndef ONEFLOW_CORE_COMM_NETWORK_RDMA_RDMA_MEMORY_H
#ifndef ONEFLOW_CORE_COMM_NETWORK_RDMA_RDMA_MEMORY_H
_
#define ONEFLOW_CORE_COMM_NETWORK_RDMA_RDMA_MEMORY_H
#define ONEFLOW_CORE_COMM_NETWORK_RDMA_RDMA_MEMORY_H
_
#include "oneflow/core/common/util.h"
#include "oneflow/core/common/util.h"
...
@@ -27,4 +27,4 @@ class RdmaMem {
...
@@ -27,4 +27,4 @@ class RdmaMem {
#endif // WITH_RDMA
#endif // WITH_RDMA
#endif // ONEFLOW_CORE_COMM_NETWORK_RDMA_RDMA_MEMORY_H
#endif // ONEFLOW_CORE_COMM_NETWORK_RDMA_RDMA_MEMORY_H
_
oneflow/core/graph/backward_compute_task_node.h
浏览文件 @
f5e8efae
...
@@ -9,7 +9,7 @@ class BackwardCompTaskNode : public CompTaskNode {
...
@@ -9,7 +9,7 @@ class BackwardCompTaskNode : public CompTaskNode {
public:
public:
OF_DISALLOW_COPY_AND_MOVE
(
BackwardCompTaskNode
);
OF_DISALLOW_COPY_AND_MOVE
(
BackwardCompTaskNode
);
BackwardCompTaskNode
()
=
default
;
BackwardCompTaskNode
()
=
default
;
~
BackwardCompTaskNode
()
=
default
;
virtual
~
BackwardCompTaskNode
()
=
default
;
void
ProduceAllRegstsAndBindEdges
()
override
;
void
ProduceAllRegstsAndBindEdges
()
override
;
void
ConsumeAllRegsts
()
override
;
void
ConsumeAllRegsts
()
override
;
...
...
oneflow/core/graph/forward_compute_task_node.h
浏览文件 @
f5e8efae
...
@@ -9,7 +9,7 @@ class ForwardCompTaskNode : public CompTaskNode {
...
@@ -9,7 +9,7 @@ class ForwardCompTaskNode : public CompTaskNode {
public:
public:
OF_DISALLOW_COPY_AND_MOVE
(
ForwardCompTaskNode
);
OF_DISALLOW_COPY_AND_MOVE
(
ForwardCompTaskNode
);
ForwardCompTaskNode
()
=
default
;
ForwardCompTaskNode
()
=
default
;
~
ForwardCompTaskNode
()
=
default
;
virtual
~
ForwardCompTaskNode
()
=
default
;
void
ProduceAllRegstsAndBindEdges
()
override
;
void
ProduceAllRegstsAndBindEdges
()
override
;
void
ConsumeAllRegsts
()
override
;
void
ConsumeAllRegsts
()
override
;
...
...
oneflow/core/kernel/data_loader_kernel.cpp
→
oneflow/core/kernel/
basic_
data_loader_kernel.cpp
浏览文件 @
f5e8efae
#include "oneflow/core/kernel/data_loader_kernel.h"
#include "oneflow/core/kernel/
basic_
data_loader_kernel.h"
#include "oneflow/core/common/str_util.h"
#include "oneflow/core/common/str_util.h"
#include "oneflow/core/job/runtime_context.h"
#include "oneflow/core/job/runtime_context.h"
#include "oneflow/core/persistence/cyclic_persistent_in_stream.h"
#include "oneflow/core/persistence/cyclic_persistent_in_stream.h"
...
@@ -7,7 +7,7 @@
...
@@ -7,7 +7,7 @@
namespace
oneflow
{
namespace
oneflow
{
template
<
typename
T
>
template
<
typename
T
>
void
DataLoaderKernel
<
T
>::
Forward
(
void
Basic
DataLoaderKernel
<
T
>::
Forward
(
const
KernelCtx
&
kernel_ctx
,
const
KernelCtx
&
kernel_ctx
,
std
::
function
<
Blob
*
(
const
std
::
string
&
)
>
BnInOp2Blob
)
const
{
std
::
function
<
Blob
*
(
const
std
::
string
&
)
>
BnInOp2Blob
)
const
{
Blob
*
out_blob
=
BnInOp2Blob
(
"out"
);
Blob
*
out_blob
=
BnInOp2Blob
(
"out"
);
...
@@ -48,9 +48,9 @@ void DataLoaderKernel<T>::Forward(
...
@@ -48,9 +48,9 @@ void DataLoaderKernel<T>::Forward(
}
}
template
<
typename
T
>
template
<
typename
T
>
void
DataLoaderKernel
<
T
>::
VirtualKernelInit
(
void
Basic
DataLoaderKernel
<
T
>::
VirtualKernelInit
(
const
ParallelContext
*
parallel_ctx
)
{
const
ParallelContext
*
parallel_ctx
)
{
const
std
::
string
&
data_dir
=
op_conf
().
data_loader_conf
().
data_dir
();
const
std
::
string
&
data_dir
=
op_conf
().
basic_
data_loader_conf
().
data_dir
();
std
::
string
parallel_id
=
std
::
to_string
(
parallel_ctx
->
parallel_id
());
std
::
string
parallel_id
=
std
::
to_string
(
parallel_ctx
->
parallel_id
());
std
::
string
file_path
=
JoinPath
(
data_dir
,
"part-"
+
parallel_id
);
std
::
string
file_path
=
JoinPath
(
data_dir
,
"part-"
+
parallel_id
);
if
(
JobDesc
::
Singleton
()
->
IsTrain
())
{
if
(
JobDesc
::
Singleton
()
->
IsTrain
())
{
...
@@ -60,7 +60,7 @@ void DataLoaderKernel<T>::VirtualKernelInit(
...
@@ -60,7 +60,7 @@ void DataLoaderKernel<T>::VirtualKernelInit(
}
}
}
}
ADD_CPU_DEFAULT_KERNEL_CREATOR
(
OperatorConf
::
k
DataLoaderConf
,
DataLoaderKernel
,
ADD_CPU_DEFAULT_KERNEL_CREATOR
(
OperatorConf
::
k
BasicDataLoaderConf
,
ARITHMETIC_DATA_TYPE_SEQ
);
BasicDataLoaderKernel
,
ARITHMETIC_DATA_TYPE_SEQ
);
}
// namespace oneflow
}
// namespace oneflow
oneflow/core/kernel/data_loader_kernel.h
→
oneflow/core/kernel/
basic_
data_loader_kernel.h
浏览文件 @
f5e8efae
#ifndef ONEFLOW_CORE_KERNEL_DATA_LOADER_KERNEL_H_
#ifndef ONEFLOW_CORE_KERNEL_
BASIC_
DATA_LOADER_KERNEL_H_
#define ONEFLOW_CORE_KERNEL_DATA_LOADER_KERNEL_H_
#define ONEFLOW_CORE_KERNEL_
BASIC_
DATA_LOADER_KERNEL_H_
#include "oneflow/core/kernel/kernel.h"
#include "oneflow/core/kernel/kernel.h"
namespace
oneflow
{
namespace
oneflow
{
template
<
typename
T
>
template
<
typename
T
>
class
DataLoaderKernel
final
:
public
KernelIf
<
DeviceType
::
kCPU
>
{
class
Basic
DataLoaderKernel
final
:
public
KernelIf
<
DeviceType
::
kCPU
>
{
public:
public:
OF_DISALLOW_COPY_AND_MOVE
(
DataLoaderKernel
);
OF_DISALLOW_COPY_AND_MOVE
(
Basic
DataLoaderKernel
);
DataLoaderKernel
()
=
default
;
Basic
DataLoaderKernel
()
=
default
;
~
DataLoaderKernel
()
=
default
;
~
Basic
DataLoaderKernel
()
=
default
;
void
Forward
(
const
KernelCtx
&
,
void
Forward
(
const
KernelCtx
&
,
std
::
function
<
Blob
*
(
const
std
::
string
&
)
>
)
const
override
;
std
::
function
<
Blob
*
(
const
std
::
string
&
)
>
)
const
override
;
...
@@ -23,4 +23,4 @@ class DataLoaderKernel final : public KernelIf<DeviceType::kCPU> {
...
@@ -23,4 +23,4 @@ class DataLoaderKernel final : public KernelIf<DeviceType::kCPU> {
}
// namespace oneflow
}
// namespace oneflow
#endif // ONEFLOW_CORE_KERNEL_DATA_LOADER_KERNEL_H_
#endif // ONEFLOW_CORE_KERNEL_
BASIC_
DATA_LOADER_KERNEL_H_
oneflow/core/operator/data_loader_op.cpp
→
oneflow/core/operator/
basic_
data_loader_op.cpp
浏览文件 @
f5e8efae
#include "oneflow/core/operator/data_loader_op.h"
#include "oneflow/core/operator/
basic_
data_loader_op.h"
#include "oneflow/core/job/job_desc.h"
#include "oneflow/core/job/job_desc.h"
namespace
oneflow
{
namespace
oneflow
{
void
DataLoaderOp
::
InitFromOpConf
()
{
void
Basic
DataLoaderOp
::
InitFromOpConf
()
{
CHECK
(
op_conf
().
has_data_loader_conf
());
CHECK
(
op_conf
().
has_
basic_
data_loader_conf
());
EnrollOutputBn
(
"out"
,
false
);
EnrollOutputBn
(
"out"
,
false
);
}
}
const
PbMessage
&
DataLoaderOp
::
GetSpecialConf
()
const
{
const
PbMessage
&
Basic
DataLoaderOp
::
GetSpecialConf
()
const
{
return
op_conf
().
data_loader_conf
();
return
op_conf
().
basic_
data_loader_conf
();
}
}
void
DataLoaderOp
::
InferBlobDescs
(
void
Basic
DataLoaderOp
::
InferBlobDescs
(
std
::
function
<
BlobDesc
*
(
const
std
::
string
)
>
GetBlobDesc4BnInOp
,
std
::
function
<
BlobDesc
*
(
const
std
::
string
)
>
GetBlobDesc4BnInOp
,
const
ParallelContext
*
parallel_ctx
)
const
{
const
ParallelContext
*
parallel_ctx
)
const
{
const
DataLoaderOpConf
&
conf
=
op_conf
().
data_loader_conf
();
const
BasicDataLoaderOpConf
&
conf
=
op_conf
().
basic_
data_loader_conf
();
BlobDesc
*
out
=
GetBlobDesc4BnInOp
(
"out"
);
BlobDesc
*
out
=
GetBlobDesc4BnInOp
(
"out"
);
std
::
vector
<
int64_t
>
dim_vec
(
1
+
conf
.
shape
().
dim_size
());
std
::
vector
<
int64_t
>
dim_vec
(
1
+
conf
.
shape
().
dim_size
());
dim_vec
[
0
]
=
JobDesc
::
Singleton
()
->
SinglePieceSize
();
dim_vec
[
0
]
=
JobDesc
::
Singleton
()
->
SinglePieceSize
();
...
@@ -28,6 +28,6 @@ void DataLoaderOp::InferBlobDescs(
...
@@ -28,6 +28,6 @@ void DataLoaderOp::InferBlobDescs(
out
->
set_has_data_id
(
JobDesc
::
Singleton
()
->
SizeOfOneDataId
()
>
0
);
out
->
set_has_data_id
(
JobDesc
::
Singleton
()
->
SizeOfOneDataId
()
>
0
);
}
}
REGISTER_OP
(
OperatorConf
::
k
DataLoaderConf
,
DataLoaderOp
);
REGISTER_OP
(
OperatorConf
::
k
BasicDataLoaderConf
,
Basic
DataLoaderOp
);
}
// namespace oneflow
}
// namespace oneflow
oneflow/core/operator/data_loader_op.h
→
oneflow/core/operator/
basic_
data_loader_op.h
浏览文件 @
f5e8efae
#ifndef ONEFLOW_CORE_OPERATOR_DATA_LOADER_OP_H_
#ifndef ONEFLOW_CORE_OPERATOR_
BASIC_
DATA_LOADER_OP_H_
#define ONEFLOW_CORE_OPERATOR_DATA_LOADER_OP_H_
#define ONEFLOW_CORE_OPERATOR_
BASIC_
DATA_LOADER_OP_H_
#include "oneflow/core/operator/operator.h"
#include "oneflow/core/operator/operator.h"
namespace
oneflow
{
namespace
oneflow
{
class
DataLoaderOp
final
:
public
Operator
{
class
Basic
DataLoaderOp
final
:
public
Operator
{
public:
public:
OF_DISALLOW_COPY_AND_MOVE
(
DataLoaderOp
);
OF_DISALLOW_COPY_AND_MOVE
(
Basic
DataLoaderOp
);
DataLoaderOp
()
=
default
;
Basic
DataLoaderOp
()
=
default
;
~
DataLoaderOp
()
=
default
;
~
Basic
DataLoaderOp
()
=
default
;
void
InitFromOpConf
()
override
;
void
InitFromOpConf
()
override
;
const
PbMessage
&
GetSpecialConf
()
const
override
;
const
PbMessage
&
GetSpecialConf
()
const
override
;
...
@@ -22,4 +22,4 @@ class DataLoaderOp final : public Operator {
...
@@ -22,4 +22,4 @@ class DataLoaderOp final : public Operator {
}
// namespace oneflow
}
// namespace oneflow
#endif // ONEFLOW_CORE_OPERATOR_DATA_LOADER_OP_H_
#endif // ONEFLOW_CORE_OPERATOR_
BASIC_
DATA_LOADER_OP_H_
oneflow/core/operator/op_conf.proto
浏览文件 @
f5e8efae
...
@@ -57,7 +57,7 @@ message InnerProductOpConf {
...
@@ -57,7 +57,7 @@ message InnerProductOpConf {
optional
FillConf
bias_fill
=
6
;
optional
FillConf
bias_fill
=
6
;
}
}
message
DataLoaderOpConf
{
message
Basic
DataLoaderOpConf
{
required
string
out
=
1
;
required
string
out
=
1
;
required
DataType
data_type
=
2
;
required
DataType
data_type
=
2
;
required
string
data_dir
=
3
;
required
string
data_dir
=
3
;
...
@@ -217,7 +217,7 @@ message OperatorConf {
...
@@ -217,7 +217,7 @@ message OperatorConf {
oneof
op_type
{
oneof
op_type
{
ConvolutionOpConf
convolution_conf
=
100
;
ConvolutionOpConf
convolution_conf
=
100
;
InnerProductOpConf
innerproduct_conf
=
101
;
InnerProductOpConf
innerproduct_conf
=
101
;
DataLoaderOpConf
data_loader_conf
=
102
;
BasicDataLoaderOpConf
basic_
data_loader_conf
=
102
;
PoolingOpConf
pooling_conf
=
103
;
PoolingOpConf
pooling_conf
=
103
;
ReluOpConf
relu_conf
=
104
;
ReluOpConf
relu_conf
=
104
;
SoftmaxOpConf
softmax_conf
=
105
;
SoftmaxOpConf
softmax_conf
=
105
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录