Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 2 年 前同步成功

通知 2325
Star 20933
Fork 5424
P
Paddle
未验证 提交 7097630f 编写于 作者: N Nyakku Shigure 提交者： GitHub
浏览文件
操作

[CodeStyle][black] use black instead of yapf (#46014) 

* update config

* re-blacken python code

* temporarily disable date and diff_py_file

* skip a format
上级 016766cc
展开全部 隐藏空白更改
内联 并排
Showing with 91727 addition and 59139 deletion

要显示的变更太多。

To preserve performance only 1000 of 1000+ files are displayed.
文本差异 电子邮件补丁
.cmake-format.py
浏览文件 @ 7097630f
......@@ -109,5 +109,5 @@ with section("parse"):
"SRCS": '*',
"DEPS": '*',
}
}
},
}
.pre-commit-config.yaml
浏览文件 @ 7097630f
......@@ -28,11 +28,17 @@ repos:
python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py|
python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py
)$
- repo: https://github.com/google/yapf
rev: v0.32.0
- repo: https://github.com/psf/black.git
rev: 22.8.0
hooks:
- id: yapf
files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
- id: black
files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
# Temporary exclude, will be formatted in a separate PR
exclude: |
(?x)^(
python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py|
python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py
)$
- repo: https://github.com/PyCQA/flake8
rev: 4.0.1
hooks:
......
.style.yapf 已删除 100644 → 0
浏览文件 @ 016766cc
[style]
based_on_style = pep8
column_limit = 80
cmake/copyfile.py
浏览文件 @ 7097630f
......@@ -21,7 +21,7 @@ import glob
def main():
src = sys.argv[1]
dst = sys.argv[2]
if os.path.isdir(src): #copy directory
if os.path.isdir(src): # copy directory
pathList = os.path.split(src)
dst = os.path.join(dst, pathList[-1])
if not os.path.exists(dst):
......@@ -31,7 +31,7 @@ def main():
shutil.rmtree(dst)
shutil.copytree(src, dst)
print("overwritten copy directory: {0} --->>> {1}".format(src, dst))
else: #copy file, wildcard
else: # copy file, wildcard
if not os.path.exists(dst):
os.makedirs(dst)
srcFiles = glob.glob(src)
......
cmake/make_resource.py
浏览文件 @ 7097630f
......@@ -20,7 +20,15 @@ res = sys.argv[1]
out = sys.argv[2]
var = re.sub(r'[ .-]', '_', os.path.basename(res))
open(out, "w").write("const unsigned char " + var + "[] = {" +
",".join(["0x%02x" % ord(c)
for c in open(res).read()]) + ",0};\n" +
"const unsigned " + var + "_size = sizeof(" + var + ");\n")
open(out, "w").write(
"const unsigned char "
+ var
+ "[] = {"
+ ",".join(["0x%02x" % ord(c) for c in open(res).read()])
+ ",0};\n"
+ "const unsigned "
+ var
+ "_size = sizeof("
+ var
+ ");\n"
)
paddle/fluid/eager/auto_code_generator/generate_file_structures.py
浏览文件 @ 7097630f
......@@ -41,8 +41,9 @@ def GenerateFileStructureForFinalDygraph(eager_dir):
os.mkdir(directory)
# Empty files
dygraph_forward_api_h_path = os.path.join(generated_dir,
"dygraph_functions.h")
dygraph_forward_api_h_path = os.path.join(
generated_dir, "dygraph_functions.h"
)
empty_files = [dygraph_forward_api_h_path]
empty_files.append(os.path.join(forwards_dir, "dygraph_functions.cc"))
empty_files.append(os.path.join(nodes_dir, "nodes.cc"))
......@@ -83,32 +84,41 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count):
os.mkdir(directory)
# Empty files
dygraph_forward_api_h_path = os.path.join(generated_dir,
"dygraph_forward_api.h")
dygraph_forward_api_h_path = os.path.join(
generated_dir, "dygraph_forward_api.h"
)
empty_files = [dygraph_forward_api_h_path]
empty_files.append(os.path.join(nodes_dir, "nodes.h"))
for i in range(split_count):
empty_files.append(
os.path.join(forwards_dir,
"dygraph_forward_functions" + str(i + 1) + ".cc"))
empty_files.append(os.path.join(nodes_dir,
"nodes" + str(i + 1) + ".cc"))
os.path.join(
forwards_dir, "dygraph_forward_functions" + str(i + 1) + ".cc"
)
)
empty_files.append(
os.path.join(nodes_dir, "nodes" + str(i + 1) + ".cc")
)
empty_files.append(
os.path.join(forwards_dir, "dygraph_forward_functions_args_info.cc"))
os.path.join(forwards_dir, "dygraph_forward_functions_args_info.cc")
)
empty_files.append(
os.path.join(forwards_dir,
"dygraph_forward_functions_args_type_info.cc"))
os.path.join(
forwards_dir, "dygraph_forward_functions_args_type_info.cc"
)
)
empty_files.append(
os.path.join(forwards_dir, "dygraph_forward_functions_returns_info.cc"))
os.path.join(forwards_dir, "dygraph_forward_functions_returns_info.cc")
)
for path in empty_files:
if not os.path.exists(path):
open(path, 'a').close()
# CMakeLists
nodes_level_cmakelist_path = os.path.join(nodes_dir, "CMakeLists.txt")
generated_level_cmakelist_path = os.path.join(generated_dir,
"CMakeLists.txt")
generated_level_cmakelist_path = os.path.join(
generated_dir, "CMakeLists.txt"
)
forwards_level_cmakelist_path = os.path.join(forwards_dir, "CMakeLists.txt")
with open(nodes_level_cmakelist_path, "w") as f:
......@@ -120,9 +130,11 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count):
for i in range(split_count):
f.write(
" COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes"
+ str(i + 1) +
".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes"
+ str(i + 1) + ".cc\"\n")
+ str(i + 1)
+ ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes"
+ str(i + 1)
+ ".cc\"\n"
)
f.write(" DEPENDS legacy_eager_codegen\n")
f.write(" VERBATIM)\n")
......@@ -142,9 +154,11 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count):
for i in range(split_count):
f.write(
" COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions"
+ str(i + 1) +
".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions"
+ str(i + 1) + ".cc\"\n")
+ str(i + 1)
+ ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions"
+ str(i + 1)
+ ".cc\"\n"
)
f.write(
" COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions_args_info.tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions_args_info.cc\"\n"
)
......@@ -167,7 +181,8 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count):
"${fluid_manual_functions} DEPS ${eager_deps} ${fluid_deps} ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})\n"
)
f.write(
"add_dependencies(dygraph_function copy_dygraph_forward_functions)")
"add_dependencies(dygraph_function copy_dygraph_forward_functions)"
)
with open(generated_level_cmakelist_path, "w") as f:
f.write("add_subdirectory(forwards)\nadd_subdirectory(nodes)")
......
paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py
浏览文件 @ 7097630f
......@@ -18,20 +18,45 @@ import re
########################
### Global Variables ###
########################
ops_to_fill_zero_for_empty_grads = set([
"split_grad", "split_with_num_grad", "rnn_grad", "matmul_double_grad",
"matmul_triple_grad", "sigmoid_double_grad", "sigmoid_triple_grad",
"add_double_grad", "add_triple_grad", "multiply_grad",
"multiply_double_grad", "multiply_triple_grad", "conv2d_grad_grad",
"batch_norm_double_grad", "tanh_double_grad", "tanh_triple_grad",
"subtract_double_grad", "divide_double_grad", "log_double_grad",
"elu_double_grad", "leaky_relu_double_grad", "sqrt_double_grad",
"rsqrt_double_grad", "square_double_grad", "celu_double_grad",
"pad_double_grad", "pad3d_double_grad", "squeeze_double_grad",
"unsqueeze_double_grad", "instance_norm_double_grad", "conv3d_double_grad",
"depthwise_conv2d_grad_grad", "concat_double_grad", "expand_grad",
"argsort_grad"
])
ops_to_fill_zero_for_empty_grads = set(
[
"split_grad",
"split_with_num_grad",
"rnn_grad",
"matmul_double_grad",
"matmul_triple_grad",
"sigmoid_double_grad",
"sigmoid_triple_grad",
"add_double_grad",
"add_triple_grad",
"multiply_grad",
"multiply_double_grad",
"multiply_triple_grad",
"conv2d_grad_grad",
"batch_norm_double_grad",
"tanh_double_grad",
"tanh_triple_grad",
"subtract_double_grad",
"divide_double_grad",
"log_double_grad",
"elu_double_grad",
"leaky_relu_double_grad",
"sqrt_double_grad",
"rsqrt_double_grad",
"square_double_grad",
"celu_double_grad",
"pad_double_grad",
"pad3d_double_grad",
"squeeze_double_grad",
"unsqueeze_double_grad",
"instance_norm_double_grad",
"conv3d_double_grad",
"depthwise_conv2d_grad_grad",
"concat_double_grad",
"expand_grad",
"argsort_grad",
]
)
# For API dispatch used at python-level
# { op_name : [arg_name, ...] }
......@@ -40,22 +65,31 @@ core_ops_args_info = {}
core_ops_args_type_info = {}
yaml_types_mapping = {
'int' : 'int', 'int32_t' : 'int32_t', 'int64_t' : 'int64_t', 'size_t' : 'size_t', \
'float' : 'float', 'double' : 'double', 'bool' : 'bool', \
'str' : 'std::string', \
'str[]' : 'std::vector<std::string>', 'float[]' : 'std::vector<float>', \
'Place' : 'paddle::Place', 'DataLayout' : 'phi::DataLayout', 'DataType' : 'paddle::experimental::DataType', \
'int64_t[]' : 'std::vector<int64_t>', 'int[]' : 'std::vector<int>',
'Tensor' : 'Tensor',
'Tensor[]' : 'std::vector<Tensor>',
'Tensor[Tensor[]]' : 'std::vector<std::vector<Tensor>>',
'Scalar' : 'paddle::experimental::Scalar',
'Scalar(int)' : 'paddle::experimental::Scalar',
'Scalar(int64_t)' : 'paddle::experimental::Scalar',
'Scalar(float)' : 'paddle::experimental::Scalar',
'Scalar(double)' : 'paddle::experimental::Scalar',
'Scalar[]' : 'std::vector<phi::Scalar>',
'IntArray' : 'paddle::experimental::IntArray'
'int': 'int',
'int32_t': 'int32_t',
'int64_t': 'int64_t',
'size_t': 'size_t',
'float': 'float',
'double': 'double',
'bool': 'bool',
'str': 'std::string',
'str[]': 'std::vector<std::string>',
'float[]': 'std::vector<float>',
'Place': 'paddle::Place',
'DataLayout': 'phi::DataLayout',
'DataType': 'paddle::experimental::DataType',
'int64_t[]': 'std::vector<int64_t>',
'int[]': 'std::vector<int>',
'Tensor': 'Tensor',
'Tensor[]': 'std::vector<Tensor>',
'Tensor[Tensor[]]': 'std::vector<std::vector<Tensor>>',
'Scalar': 'paddle::experimental::Scalar',
'Scalar(int)': 'paddle::experimental::Scalar',
'Scalar(int64_t)': 'paddle::experimental::Scalar',
'Scalar(float)': 'paddle::experimental::Scalar',
'Scalar(double)': 'paddle::experimental::Scalar',
'Scalar[]': 'std::vector<phi::Scalar>',
'IntArray': 'paddle::experimental::IntArray',
}
......@@ -81,7 +115,8 @@ def ReadBwdFile(filepath):
if contents is not None:
for content in contents:
assert 'backward_op' in content.keys(), AssertMessage(
'backward_op', content.keys())
'backward_op', content.keys()
)
if 'backward_op' in content.keys():
api_name = content['backward_op']
......@@ -116,7 +151,8 @@ def IsPlainTensorType(string):
def IsVectorTensorType(string):
vector_tensor_types = [
'std::vector<std::vector<Tensor>>', 'std::vector<Tensor>'
'std::vector<std::vector<Tensor>>',
'std::vector<Tensor>',
]
if string in vector_tensor_types:
return True
......@@ -147,7 +183,6 @@ def RemoveConstAndReference(string):
def GetGradNodeName(string):
def str2Hump(text):
arr = filter(None, text.split('_'))
res = ''
......@@ -166,7 +201,6 @@ def GetDygraphForwardFunctionName(string):
def GetDygraphLogName(string):
def str2Hump(text):
arr = filter(None, text.split('_'))
res = ''
......@@ -236,10 +270,14 @@ def ParseYamlArgs(string):
m = re.search(pattern, arg)
arg_type = m.group(1).strip()
arg_name = m.group(3).split("=")[0].strip()
default_value = m.group(3).split("=")[1].strip() if len(
m.group(3).split("=")) > 1 else None
assert arg_type in yaml_types_mapping.keys(
default_value = (
m.group(3).split("=")[1].strip()
if len(m.group(3).split("=")) > 1
else None
)
assert (
arg_type in yaml_types_mapping.keys()
), f"The argument type {arg_type} in yaml config is not supported in yaml_types_mapping."
if arg_type in ["DataType", "DataLayout"] and default_value is not None:
default_value = f"paddle::experimental::{default_value}"
......@@ -277,7 +315,8 @@ def ParseYamlReturns(string):
else:
ret_type = ret.strip()
assert ret_type in yaml_types_mapping.keys(
assert (
ret_type in yaml_types_mapping.keys()
), f"The return type {ret_type} in yaml config is not supported in yaml_types_mapping."
ret_type = yaml_types_mapping[ret_type]
......@@ -295,7 +334,9 @@ def ParseYamlForwardFromBackward(string):
wspace = r'\s*'
fargs = r'(.*?)'
frets = r'(.*)'
pattern = fr'{fname}{wspace}\({wspace}{fargs}{wspace}\){wspace}->{wspace}{frets}'
pattern = (
fr'{fname}{wspace}\({wspace}{fargs}{wspace}\){wspace}->{wspace}{frets}'
)
m = re.search(pattern, string)
function_name = m.group(1)
......@@ -359,38 +400,44 @@ def ParseYamlInplaceInfo(string):
### Generator Base ###
########################
class FunctionGeneratorBase:
def __init__(self, forward_api_contents, namespace):
self.forward_api_contents = forward_api_contents
self.namespace = namespace
self.is_forward_only = False if 'backward' in forward_api_contents.keys(
) else True
self.is_forward_only = (
False if 'backward' in forward_api_contents.keys() else True
)
self.forward_api_name = ""
self.orig_forward_inputs_list = [
] #[ [arg_name, arg_type, orig_position], ...]
self.orig_forward_attrs_list = [
] #[ [attr_name, attr_type, default_value, orig_position], ...]
self.orig_forward_returns_list = [
] #[ [ret_name, ret_type, orig_position], ...]
self.orig_forward_inputs_list = (
[]
) # [ [arg_name, arg_type, orig_position], ...]
self.orig_forward_attrs_list = (
[]
) # [ [attr_name, attr_type, default_value, orig_position], ...]
self.orig_forward_returns_list = (
[]
) # [ [ret_name, ret_type, orig_position], ...]
# Processed Forward Data
self.forward_inputs_position_map = {
} #{ "name" : [type, fwd_position] }
self.forward_outputs_position_map = {
} #{ "name" : [type, fwd_position] }
self.forward_inputs_position_map = (
{}
) # { "name" : [type, fwd_position] }
self.forward_outputs_position_map = (
{}
) # { "name" : [type, fwd_position] }
# Special Op Attributes
self.optional_inputs = [] #[name, ...]
self.no_need_buffers = [] #[name, ...]
self.intermediate_outputs = [] #[name, ...]
self.forward_inplace_map = {} #{name : name, ...}
self.optional_inputs = [] # [name, ...]
self.no_need_buffers = [] # [name, ...]
self.intermediate_outputs = [] # [name, ...]
self.forward_inplace_map = {} # {name : name, ...}
def ParseForwardInplaceInfo(self):
forward_api_contents = self.forward_api_contents
if 'inplace' not in forward_api_contents.keys(): return
if 'inplace' not in forward_api_contents.keys():
return
inplace_map_str = forward_api_contents['inplace']
self.forward_inplace_map = ParseYamlInplaceInfo(inplace_map_str)
......@@ -432,19 +479,26 @@ class FunctionGeneratorBase:
forward_args_str = forward_api_contents['args']
forward_returns_str = forward_api_contents['output']
assert 'op' in forward_api_contents.keys(
assert (
'op' in forward_api_contents.keys()
), "Unable to find \"op\" in forward_api_contents keys"
assert 'args' in forward_api_contents.keys(
assert (
'args' in forward_api_contents.keys()
), "Unable to find \"args\" in forward_api_contents keys"
assert 'output' in forward_api_contents.keys(
assert (
'output' in forward_api_contents.keys()
), "Unable to find \"output\" in forward_api_contents keys"
# Collect Original Forward Inputs/Outputs and then perform validation checks
self.orig_forward_inputs_list, self.orig_forward_attrs_list, self.orig_forward_returns_list = ParseYamlForward(
forward_args_str, forward_returns_str)
def DetermineForwardPositionMap(self, forward_inputs_list,
forward_returns_list):
(
self.orig_forward_inputs_list,
self.orig_forward_attrs_list,
self.orig_forward_returns_list,
) = ParseYamlForward(forward_args_str, forward_returns_str)
def DetermineForwardPositionMap(
self, forward_inputs_list, forward_returns_list
):
for i in range(len(forward_inputs_list)):
forward_input = forward_inputs_list[i]
input_name = forward_input[0]
......@@ -452,13 +506,14 @@ class FunctionGeneratorBase:
input_pos = forward_input[2]
self.forward_inputs_position_map[input_name] = [
input_type, input_pos
input_type,
input_pos,
]
for i in range(len(forward_returns_list)):
forward_return = forward_returns_list[i]
if len(forward_return[0]) == 0:
if (len(forward_returns_list) == 1):
if len(forward_returns_list) == 1:
return_name = "out"
else:
return_name = "out_{}".format(i + 1)
......@@ -468,12 +523,12 @@ class FunctionGeneratorBase:
return_pos = forward_return[2]
self.forward_outputs_position_map[return_name] = [
return_type, return_pos
return_type,
return_pos,
]
class GeneratorBase:
def __init__(self, api_yaml_path):
self.namespace = ""
self.api_yaml_path = api_yaml_path
......
paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py
浏览文件 @ 7097630f
......@@ -94,8 +94,9 @@ def download_concat(cache_folder, zip_path):
def print_processbar(done_percentage):
done_filled = done_percentage * '='
empty_filled = (100 - done_percentage) * ' '
sys.stdout.write("\r[%s%s]%d%%" %
(done_filled, empty_filled, done_percentage))
sys.stdout.write(
"\r[%s%s]%d%%" % (done_filled, empty_filled, done_percentage)
)
sys.stdout.flush()
......@@ -137,7 +138,7 @@ def convert_Imagenet_tar2bin(tar_file, output_file):
val_dict[name] = label
for img_name in dataset.keys():
remove_len = (len(FOLDER_NAME))
remove_len = len(FOLDER_NAME)
img_name_prim = img_name[remove_len:]
label = val_dict[img_name_prim]
label_int = (int)(label)
......@@ -156,19 +157,25 @@ def run_convert():
retry = 0
try_limit = 3
while not (os.path.exists(output_file)
and os.path.getsize(output_file) == FULL_SIZE_BYTES):
while not (
os.path.exists(output_file)
and os.path.getsize(output_file) == FULL_SIZE_BYTES
):
if os.path.exists(output_file):
sys.stderr.write(
"\n\nThe existing binary file[{}] is broken. Start to generate new one...\n\n"
.format(output_file))
"\n\nThe existing binary file[{}] is broken. Start to generate new one...\n\n".format(
output_file
)
)
os.remove(output_file)
if retry < try_limit:
retry = retry + 1
else:
raise RuntimeError(
"Can not convert the dataset to binary file with try limit {0}".
format(try_limit))
"Can not convert the dataset to binary file with try limit {0}".format(
try_limit
)
)
download_concat(cache_folder, zip_path)
convert_Imagenet_tar2bin(zip_path, output_file)
print("\nSuccess! The binary file can be found at {0}".format(output_file))
......@@ -193,63 +200,75 @@ def convert_Imagenet_local2bin(args):
if not os.path.exists(img_path):
continue
#save image(float32) to file
# save image(float32) to file
img = Image.open(img_path)
img = process_image(img)
np_img = np.array(img)
of.seek(SIZE_INT64 +
SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx)
of.seek(
SIZE_INT64 + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx
)
of.write(np_img.astype('float32').tobytes())
#save label(int64_t) to file
# save label(int64_t) to file
label_int = (int)(label)
np_label = np.array(label_int)
of.seek(SIZE_INT64 +
SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images +
idx * SIZE_INT64)
of.seek(
SIZE_INT64
+ SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images
+ idx * SIZE_INT64
)
of.write(np_label.astype('int64').tobytes())
# The bin file should contain
# number of images + all images data + all corresponding labels
# so the file target_size should be as follows
target_size = SIZE_INT64 + num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32 + num_images * SIZE_INT64
if (os.path.getsize(bin_file_path) == target_size):
target_size = (
SIZE_INT64
+ num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32
+ num_images * SIZE_INT64
)
if os.path.getsize(bin_file_path) == target_size:
print(
"Success! The user data output binary file can be found at: {0}"
.format(bin_file_path))
"Success! The user data output binary file can be found at: {0}".format(
bin_file_path
)
)
else:
print("Conversion failed!")
def main_preprocess_Imagenet(args):
parser = argparse.ArgumentParser(
description=
"Convert the full Imagenet val set or local data to binary file.",
description="Convert the full Imagenet val set or local data to binary file.",
usage=None,
add_help=True)
add_help=True,
)
parser.add_argument(
'--local',
action="store_true",
help="If used, user need to set --data_dir and then convert file")
parser.add_argument("--data_dir",
default="",
type=str,
help="Dataset root directory")
help="If used, user need to set --data_dir and then convert file",
)
parser.add_argument(
"--data_dir", default="", type=str, help="Dataset root directory"
)
parser.add_argument(
"--label_list",
type=str,
default="val_list.txt",
help=
"List of object labels with same sequence as denoted in the annotation file"
help="List of object labels with same sequence as denoted in the annotation file",
)
parser.add_argument(
"--output_file",
type=str,
default="imagenet_small.bin",
help="File path of the output binary file",
)
parser.add_argument(
"--data_dim",
type=int,
default=DATA_DIM,
help="Image preprocess with data_dim width and height",
)
parser.add_argument("--output_file",
type=str,
default="imagenet_small.bin",
help="File path of the output binary file")
parser.add_argument("--data_dim",
type=int,
default=DATA_DIM,
help="Image preprocess with data_dim width and height")
args = parser.parse_args()
if args.local:
......
paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py
浏览文件 @ 7097630f
......@@ -24,7 +24,9 @@ import hashlib
import tarfile
import argparse
DATA_URL = "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar"
DATA_URL = (
"http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar"
)
DATA_DIR = os.path.expanduser("~/.cache/paddle/dataset/pascalvoc/")
TAR_FILE = "VOCtest_06-Nov-2007.tar"
TAR_PATH = os.path.join(DATA_DIR, TAR_FILE)
......@@ -106,8 +108,9 @@ def convert_pascalvoc_local2bin(args):
for object in objects:
bbox_sample = []
# start from 1
bbox_sample.append(float(label_list.index(
object.find('name').text)))
bbox_sample.append(
float(label_list.index(object.find('name').text))
)
bbox = object.find('bndbox')
difficult = float(object.find('difficult').text)
bbox_sample.append(float(bbox.find('xmin').text) / im_width)
......@@ -118,7 +121,8 @@ def convert_pascalvoc_local2bin(args):
bbox_labels.append(bbox_sample)
bbox_labels = np.array(bbox_labels)
if len(bbox_labels) == 0: continue
if len(bbox_labels) == 0:
continue
lbls.extend(bbox_labels[:, 0])
boxes.extend(bbox_labels[:, 1:5])
......@@ -135,11 +139,17 @@ def convert_pascalvoc_local2bin(args):
# number of images + all images data + an array that represent object numbers of each image
# + labels of all objects in images + bboxes of all objects + difficulties of all objects
# so the target size should be as follows:
target_size = SIZE_INT64 + image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32 + image_nums * SIZE_INT64 + object_nums_sum * (
SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64)
if (os.path.getsize(output_file_path) == target_size):
print("Success! \nThe local data output binary file can be found at: ",
output_file_path)
target_size = (
SIZE_INT64
+ image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32
+ image_nums * SIZE_INT64
+ object_nums_sum * (SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64)
)
if os.path.getsize(output_file_path) == target_size:
print(
"Success! \nThe local data output binary file can be found at: ",
output_file_path,
)
else:
print("Conversion failed!")
......@@ -147,8 +157,9 @@ def convert_pascalvoc_local2bin(args):
def print_processbar(done_percentage):
done_filled = done_percentage * '='
empty_filled = (100 - done_percentage) * ' '
sys.stdout.write("\r[%s%s]%d%%" %
(done_filled, empty_filled, done_percentage))
sys.stdout.write(
"\r[%s%s]%d%%" % (done_filled, empty_filled, done_percentage)
)
sys.stdout.flush()
......@@ -163,13 +174,30 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path):
# map label to number (index)
label_list = [
"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus",
"car", "cat", "chair", "cow", "diningtable", "dog", "horse",
"motorbike", "person", "pottedplant", "sheep", "sofa", "train",
"tvmonitor"
"background",
"aeroplane",
"bicycle",
"bird",
"boat",
"bottle",
"bus",
"car",
"cat",
"chair",
"cow",
"diningtable",
"dog",
"horse",
"motorbike",
"person",
"pottedplant",
"sheep",
"sofa",
"train",
"tvmonitor",
]
print_processbar(0)
#read from tar file and write to bin
# read from tar file and write to bin
tar = tarfile.open(tar_path, "r")
f_test = tar.extractfile(TEST_LIST_KEY).read()
lines = f_test.split('\n')
......@@ -210,8 +238,9 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path):
for object in objects:
bbox_sample = []
bbox_sample.append(float(label_list.index(
object.find('name').text)))
bbox_sample.append(
float(label_list.index(object.find('name').text))
)
bbox = object.find('bndbox')
difficult = float(object.find('difficult').text)
bbox_sample.append(float(bbox.find('xmin').text) / im_width)
......@@ -222,7 +251,8 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path):
bbox_labels.append(bbox_sample)
bbox_labels = np.array(bbox_labels)
if len(bbox_labels) == 0: continue
if len(bbox_labels) == 0:
continue
lbls.extend(bbox_labels[:, 0])
boxes.extend(bbox_labels[:, 1:5])
difficults.extend(bbox_labels[:, -1])
......@@ -258,12 +288,16 @@ def download_pascalvoc(data_url, data_dir, tar_targethash, tar_path):
def run_convert():
try_limit = 2
retry = 0
while not (os.path.exists(DATA_OUT_PATH) and os.path.getsize(DATA_OUT_PATH)
== BIN_FULLSIZE and BIN_TARGETHASH == hashlib.md5(
open(DATA_OUT_PATH, 'rb').read()).hexdigest()):
while not (
os.path.exists(DATA_OUT_PATH)
and os.path.getsize(DATA_OUT_PATH) == BIN_FULLSIZE
and BIN_TARGETHASH
== hashlib.md5(open(DATA_OUT_PATH, 'rb').read()).hexdigest()
):
if os.path.exists(DATA_OUT_PATH):
sys.stderr.write(
"The existing binary file is broken. It is being removed...\n")
"The existing binary file is broken. It is being removed...\n"
)
os.remove(DATA_OUT_PATH)
if retry < try_limit:
retry = retry + 1
......@@ -275,52 +309,60 @@ def run_convert():
def main_pascalvoc_preprocess(args):
parser = argparse.ArgumentParser(
description=
"Convert the full pascalvoc val set or local data to binary file.",
description="Convert the full pascalvoc val set or local data to binary file.",
usage=None,
add_help=True)
add_help=True,
)
parser.add_argument(
'--local',
action="store_true",
help="If used, user need to set --data_dir and then convert file")
parser.add_argument("--data_dir",
default="",
type=str,
help="Dataset root directory")
help="If used, user need to set --data_dir and then convert file",
)
parser.add_argument(
"--data_dir", default="", type=str, help="Dataset root directory"
)
parser.add_argument(
"--img_annotation_list",
type=str,
default="test_100.txt",
help=
"A file containing the image file path and corresponding annotation file path"
help="A file containing the image file path and corresponding annotation file path",
)
parser.add_argument(
"--label_file",
type=str,
default="label_list",
help=
"List of object labels with same sequence as denoted in the annotation file"
help="List of object labels with same sequence as denoted in the annotation file",
)
parser.add_argument(
"--output_file",
type=str,
default="pascalvoc_small.bin",
help="File path of the output binary file",
)
parser.add_argument(
"--resize_h",
type=int,
default=RESIZE_H,
help="Image preprocess with resize_h",
)
parser.add_argument(
"--resize_w",
type=int,
default=RESIZE_W,
help="Image prerocess with resize_w",
)
parser.add_argument(
"--mean_value",
type=str,
default=MEAN_VALUE,
help="Image preprocess with mean_value",
)
parser.add_argument(
"--ap_version",
type=str,
default=AP_VERSION,
help="Image preprocess with ap_version",
)
parser.add_argument("--output_file",
type=str,
default="pascalvoc_small.bin",
help="File path of the output binary file")
parser.add_argument("--resize_h",
type=int,
default=RESIZE_H,
help="Image preprocess with resize_h")
parser.add_argument("--resize_w",
type=int,
default=RESIZE_W,
help="Image prerocess with resize_w")
parser.add_argument("--mean_value",
type=str,
default=MEAN_VALUE,
help="Image preprocess with mean_value")
parser.add_argument("--ap_version",
type=str,
default=AP_VERSION,
help="Image preprocess with ap_version")
args = parser.parse_args()
if args.local:
convert_pascalvoc_local2bin(args)
......
paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py
浏览文件 @ 7097630f
......@@ -17,13 +17,13 @@ import os
class Test_Preprocess(unittest.TestCase):
def test_local_convert(self):
os.system("python full_pascalvoc_test_preprocess.py --choice=local")
def test_online_convert(self):
os.system(
"python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007")
"python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007"
)
if __name__ == '__main__':
......
paddle/fluid/pybind/generate_file_structures.py
浏览文件 @ 7097630f
......@@ -25,7 +25,8 @@ if __name__ == "__main__":
for i in range(split_count):
empty_files.append(
os.path.join(pybind_dir, "op_function" + str(i + 1) + ".cc"))
os.path.join(pybind_dir, "op_function" + str(i + 1) + ".cc")
)
for path in empty_files:
if not os.path.exists(path):
......
paddle/infrt/tests/models/abs_model.py
浏览文件 @ 7097630f
......@@ -19,7 +19,6 @@ import sys
class AbsNet(paddle.nn.Layer):
def __init__(self):
super(AbsNet, self).__init__()
......@@ -32,6 +31,7 @@ if __name__ == '__main__':
# build network
model = AbsNet()
# save inferencing format model
net = to_static(model,
input_spec=[InputSpec(shape=[None, 1, 28, 28], name='x')])
net = to_static(
model, input_spec=[InputSpec(shape=[None, 1, 28, 28], name='x')]
)
paddle.jit.save(net, sys.argv[1])
paddle/infrt/tests/models/efficientnet-b4/model.py
浏览文件 @ 7097630f
......@@ -20,6 +20,7 @@ import paddle
import sys
model = EfficientNet.from_name('efficientnet-b4')
net = to_static(model,
input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')])
net = to_static(
model, input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')]
)
paddle.jit.save(net, sys.argv[1])
paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py
浏览文件 @ 7097630f
......@@ -16,9 +16,15 @@ import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from .utils import (round_filters, round_repeats, drop_connect,
get_same_padding_conv2d, get_model_params,
efficientnet_params, load_pretrained_weights)
from .utils import (
round_filters,
round_repeats,
drop_connect,
get_same_padding_conv2d,
get_model_params,
efficientnet_params,
load_pretrained_weights,
)
class MBConvBlock(nn.Layer):
......@@ -38,8 +44,9 @@ class MBConvBlock(nn.Layer):
self._block_args = block_args
self._bn_mom = global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se = (self._block_args.se_ratio
is not None) and (0 < self._block_args.se_ratio <= 1)
self.has_se = (self._block_args.se_ratio is not None) and (
0 < self._block_args.se_ratio <= 1
)
self.id_skip = block_args.id_skip # skip connection and drop connect
# Get static or dynamic convolution depending on image size
......@@ -47,15 +54,19 @@ class MBConvBlock(nn.Layer):
# Expansion phase
inp = self._block_args.input_filters # number of input channels
oup = self._block_args.input_filters * self._block_args.expand_ratio # number of output channels
oup = (
self._block_args.input_filters * self._block_args.expand_ratio
) # number of output channels
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp,
out_channels=oup,
kernel_size=1,
bias_attr=False)
self._bn0 = nn.BatchNorm2D(num_features=oup,
momentum=self._bn_mom,
epsilon=self._bn_eps)
self._expand_conv = Conv2d(
in_channels=inp,
out_channels=oup,
kernel_size=1,
bias_attr=False,
)
self._bn0 = nn.BatchNorm2D(
num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps
)
# Depthwise convolution phase
k = self._block_args.kernel_size
......@@ -66,32 +77,40 @@ class MBConvBlock(nn.Layer):
groups=oup, # groups makes it depthwise
kernel_size=k,
stride=s,
bias_attr=False)
self._bn1 = nn.BatchNorm2D(num_features=oup,
momentum=self._bn_mom,
epsilon=self._bn_eps)
bias_attr=False,
)
self._bn1 = nn.BatchNorm2D(
num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps
)
# Squeeze and Excitation layer, if desired
if self.has_se:
num_squeezed_channels = max(
1,
int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup,
out_channels=num_squeezed_channels,
kernel_size=1)
self._se_expand = Conv2d(in_channels=num_squeezed_channels,
out_channels=oup,
kernel_size=1)
int(self._block_args.input_filters * self._block_args.se_ratio),
)
self._se_reduce = Conv2d(
in_channels=oup,
out_channels=num_squeezed_channels,
kernel_size=1,
)
self._se_expand = Conv2d(
in_channels=num_squeezed_channels,
out_channels=oup,
kernel_size=1,
)
# Output phase
final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup,
out_channels=final_oup,
kernel_size=1,
bias_attr=False)
self._bn2 = nn.BatchNorm2D(num_features=final_oup,
momentum=self._bn_mom,
epsilon=self._bn_eps)
self._project_conv = Conv2d(
in_channels=oup,
out_channels=final_oup,
kernel_size=1,
bias_attr=False,
)
self._bn2 = nn.BatchNorm2D(
num_features=final_oup, momentum=self._bn_mom, epsilon=self._bn_eps
)
self._swish = nn.Hardswish()
def forward(self, inputs, drop_connect_rate=None):
......@@ -111,18 +130,26 @@ class MBConvBlock(nn.Layer):
if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
x_squeezed = self._se_expand(
self._swish(self._se_reduce(x_squeezed)))
self._swish(self._se_reduce(x_squeezed))
)
x = F.sigmoid(x_squeezed) * x
x = self._bn2(self._project_conv(x))
# Skip connection and drop connect
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
input_filters, output_filters = (
self._block_args.input_filters,
self._block_args.output_filters,
)
if (
self.id_skip
and self._block_args.stride == 1
and input_filters == output_filters
):
if drop_connect_rate:
x = drop_connect(x,
prob=drop_connect_rate,
training=self.training)
x = drop_connect(
x, prob=drop_connect_rate, training=self.training
)
x = x + inputs # skip connection
return x
......@@ -161,15 +188,14 @@ class EfficientNet(nn.Layer):
# Stem
in_channels = 3 # rgb
out_channels = round_filters(
32, self._global_params) # number of output channels
self._conv_stem = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=2,
bias_attr=False)
self._bn0 = nn.BatchNorm2D(num_features=out_channels,
momentum=bn_mom,
epsilon=bn_eps)
32, self._global_params
) # number of output channels
self._conv_stem = Conv2d(
in_channels, out_channels, kernel_size=3, stride=2, bias_attr=False
)
self._bn0 = nn.BatchNorm2D(
num_features=out_channels, momentum=bn_mom, epsilon=bn_eps
)
# Build blocks
self._blocks = nn.LayerList([])
......@@ -177,32 +203,37 @@ class EfficientNet(nn.Layer):
# Update block input and output filters based on depth multiplier.
block_args = block_args._replace(
input_filters=round_filters(block_args.input_filters,
self._global_params),
output_filters=round_filters(block_args.output_filters,
self._global_params),
num_repeat=round_repeats(block_args.num_repeat,
self._global_params))
input_filters=round_filters(
block_args.input_filters, self._global_params
),
output_filters=round_filters(
block_args.output_filters, self._global_params
),
num_repeat=round_repeats(
block_args.num_repeat, self._global_params
),
)
# The first block needs to take care of stride and filter size increase.
self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(
input_filters=block_args.output_filters, stride=1)
input_filters=block_args.output_filters, stride=1
)
for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args,
self._global_params))
self._blocks.append(
MBConvBlock(block_args, self._global_params)
)
# Head
in_channels = block_args.output_filters # output of final block
out_channels = round_filters(1280, self._global_params)
self._conv_head = Conv2d(in_channels,
out_channels,
kernel_size=1,
bias_attr=False)
self._bn1 = nn.BatchNorm2D(num_features=out_channels,
momentum=bn_mom,
epsilon=bn_eps)
self._conv_head = Conv2d(
in_channels, out_channels, kernel_size=1, bias_attr=False
)
self._bn1 = nn.BatchNorm2D(
num_features=out_channels, momentum=bn_mom, epsilon=bn_eps
)
# Final linear layer
self._avg_pooling = nn.AdaptiveAvgPool2D(1)
......@@ -217,7 +248,7 @@ class EfficientNet(nn.Layer):
block.set_swish(memory_efficient)
def extract_features(self, inputs):
""" Returns output of the final convolution layer """
"""Returns output of the final convolution layer"""
# Stem
x = self._swish(self._bn0(self._conv_stem(inputs)))
......@@ -235,7 +266,7 @@ class EfficientNet(nn.Layer):
return x
def forward(self, inputs):
""" Calls extract_features to extract features, applies final linear layer, and returns logits. """
"""Calls extract_features to extract features, applies final linear layer, and returns logits."""
bs = inputs.shape[0]
# Convolution layers
x = self.extract_features(inputs)
......@@ -250,31 +281,33 @@ class EfficientNet(nn.Layer):
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
blocks_args, global_params = get_model_params(model_name,
override_params)
blocks_args, global_params = get_model_params(
model_name, override_params
)
return cls(blocks_args, global_params)
@classmethod
def from_pretrained(cls,
model_name,
advprop=False,
num_classes=1000,
in_channels=3):
model = cls.from_name(model_name,
override_params={'num_classes': num_classes})
load_pretrained_weights(model,
model_name,
load_fc=(num_classes == 1000),
advprop=advprop)
def from_pretrained(
cls, model_name, advprop=False, num_classes=1000, in_channels=3
):
model = cls.from_name(
model_name, override_params={'num_classes': num_classes}
)
load_pretrained_weights(
model, model_name, load_fc=(num_classes == 1000), advprop=advprop
)
if in_channels != 3:
Conv2d = get_same_padding_conv2d(
image_size=model._global_params.image_size)
image_size=model._global_params.image_size
)
out_channels = round_filters(32, model._global_params)
model._conv_stem = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=2,
bias_attr=False)
model._conv_stem = Conv2d(
in_channels,
out_channels,
kernel_size=3,
stride=2,
bias_attr=False,
)
return model
@classmethod
......@@ -285,8 +318,9 @@ class EfficientNet(nn.Layer):
@classmethod
def _check_model_name_is_valid(cls, model_name):
""" Validates model name. """
"""Validates model name."""
valid_models = ['efficientnet-b' + str(i) for i in range(9)]
if model_name not in valid_models:
raise ValueError('model_name should be one of: ' +
', '.join(valid_models))
raise ValueError(
'model_name should be one of: ' + ', '.join(valid_models)
)
paddle/infrt/tests/models/efficientnet-b4/net/utils.py
浏览文件 @ 7097630f
......@@ -22,25 +22,44 @@ import paddle.nn as nn
import paddle.nn.functional as F
# Parameters for the entire model (stem, all blocks, and head)
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'num_classes',
'width_coefficient', 'depth_coefficient', 'depth_divisor', 'min_depth',
'drop_connect_rate', 'image_size'
])
GlobalParams = collections.namedtuple(
'GlobalParams',
[
'batch_norm_momentum',
'batch_norm_epsilon',
'dropout_rate',
'num_classes',
'width_coefficient',
'depth_coefficient',
'depth_divisor',
'min_depth',
'drop_connect_rate',
'image_size',
],
)
# Parameters for an individual model block
BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'
])
BlockArgs = collections.namedtuple(
'BlockArgs',
[
'kernel_size',
'num_repeat',
'input_filters',
'output_filters',
'expand_ratio',
'id_skip',
'stride',
'se_ratio',
],
)
# Change namedtuple defaults
GlobalParams.__new__.__defaults__ = (None, ) * len(GlobalParams._fields)
BlockArgs.__new__.__defaults__ = (None, ) * len(BlockArgs._fields)
GlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)
BlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)
def round_filters(filters, global_params):
""" Calculate and round number of filters based on depth multiplier. """
"""Calculate and round number of filters based on depth multiplier."""
multiplier = global_params.width_coefficient
if not multiplier:
return filters
......@@ -48,15 +67,16 @@ def round_filters(filters, global_params):
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth,
int(filters + divisor / 2) // divisor * divisor)
new_filters = max(
min_depth, int(filters + divisor / 2) // divisor * divisor
)
if new_filters < 0.9 * filters: # prevent rounding by more than 10%
new_filters += divisor
return int(new_filters)
def round_repeats(repeats, global_params):
""" Round number of filters based on depth multiplier. """
"""Round number of filters based on depth multiplier."""
multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
......@@ -76,8 +96,8 @@ def drop_connect(inputs, prob, training):
def get_same_padding_conv2d(image_size=None):
""" Chooses static padding if you have specified an image size, and dynamic padding otherwise.
Static padding is necessary for ONNX exporting of models. """
"""Chooses static padding if you have specified an image size, and dynamic padding otherwise.
Static padding is necessary for ONNX exporting of models."""
if image_size is None:
return Conv2dDynamicSamePadding
else:
......@@ -85,26 +105,31 @@ def get_same_padding_conv2d(image_size=None):
class Conv2dDynamicSamePadding(nn.Conv2D):
""" 2D Convolutions like TensorFlow, for a dynamic image size """
def __init__(self,
in_channels,
out_channels,
kernel_size,
stride=1,
dilation=1,
groups=1,
bias_attr=None):
super().__init__(in_channels,
out_channels,
kernel_size,
stride,
0,
dilation,
groups,
bias_attr=bias_attr)
self.stride = self._stride if len(
self._stride) == 2 else [self._stride[0]] * 2
"""2D Convolutions like TensorFlow, for a dynamic image size"""
def __init__(
self,
in_channels,
out_channels,
kernel_size,
stride=1,
dilation=1,
groups=1,
bias_attr=None,
):
super().__init__(
in_channels,
out_channels,
kernel_size,
stride,
0,
dilation,
groups,
bias_attr=bias_attr,
)
self.stride = (
self._stride if len(self._stride) == 2 else [self._stride[0]] * 2
)
def forward(self, x):
ih, iw = x.shape[-2:]
......@@ -112,65 +137,84 @@ class Conv2dDynamicSamePadding(nn.Conv2D):
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
pad_h = max(
(oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih,
0)
(oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, 0
)
pad_w = max(
(ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw,
0)
(ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, 0
)
if pad_h > 0 or pad_w > 0:
x = F.pad(x, [
pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2
])
return F.conv2d(x, self.weight, self.bias, self.stride, self._padding,
self._dilation, self._groups)
x = F.pad(
x,
[
pad_w // 2,
pad_w - pad_w // 2,
pad_h // 2,
pad_h - pad_h // 2,
],
)
return F.conv2d(
x,
self.weight,
self.bias,
self.stride,
self._padding,
self._dilation,
self._groups,
)
class Conv2dStaticSamePadding(nn.Conv2D):
""" 2D Convolutions like TensorFlow, for a fixed image size"""
def __init__(self,
in_channels,
out_channels,
kernel_size,
image_size=None,
**kwargs):
"""2D Convolutions like TensorFlow, for a fixed image size"""
def __init__(
self, in_channels, out_channels, kernel_size, image_size=None, **kwargs
):
if 'stride' in kwargs and isinstance(kwargs['stride'], list):
kwargs['stride'] = kwargs['stride'][0]
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
self.stride = self._stride if len(
self._stride) == 2 else [self._stride[0]] * 2
self.stride = (
self._stride if len(self._stride) == 2 else [self._stride[0]] * 2
)
# Calculate padding based on image size and save it
assert image_size is not None
ih, iw = image_size if type(image_size) == list else [
image_size, image_size
]
ih, iw = (
image_size if type(image_size) == list else [image_size, image_size]
)
kh, kw = self.weight.shape[-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
pad_h = max(
(oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih,
0)
(oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, 0
)
pad_w = max(
(ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw,
0)
(ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, 0
)
if pad_h > 0 or pad_w > 0:
self.static_padding = nn.Pad2D([
pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2
])
self.static_padding = nn.Pad2D(
[pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]
)
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
x = F.conv2d(x, self.weight, self.bias, self.stride, self._padding,
self._dilation, self._groups)
x = F.conv2d(
x,
self.weight,
self.bias,
self.stride,
self._padding,
self._dilation,
self._groups,
)
return x
class Identity(nn.Layer):
def __init__(self, ):
def __init__(
self,
):
super().__init__()
def forward(self, x):
......@@ -178,7 +222,7 @@ class Identity(nn.Layer):
def efficientnet_params(model_name):
""" Map EfficientNet model name to parameter coefficients. """
"""Map EfficientNet model name to parameter coefficients."""
params_dict = {
# Coefficients: width,depth,resolution,dropout
'efficientnet-b0': (1.0, 1.0, 224, 0.2),
......@@ -196,11 +240,11 @@ def efficientnet_params(model_name):
class BlockDecoder(object):
""" Block Decoder for readability, straight from the official TensorFlow repository """
"""Block Decoder for readability, straight from the official TensorFlow repository"""
@staticmethod
def _decode_block_string(block_string):
""" Gets a block through a string notation of arguments. """
"""Gets a block through a string notation of arguments."""
assert isinstance(block_string, str)
ops = block_string.split('_')
......@@ -212,8 +256,9 @@ class BlockDecoder(object):
options[key] = value
# Check stride
assert (('s' in options and len(options['s']) == 1) or
(len(options['s']) == 2 and options['s'][0] == options['s'][1]))
assert ('s' in options and len(options['s']) == 1) or (
len(options['s']) == 2 and options['s'][0] == options['s'][1]
)
return BlockArgs(
kernel_size=int(options['k']),
......@@ -223,7 +268,8 @@ class BlockDecoder(object):
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se']) if 'se' in options else None,
stride=[int(options['s'][0])])
stride=[int(options['s'][0])],
)
@staticmethod
def _encode_block_string(block):
......@@ -234,7 +280,7 @@ class BlockDecoder(object):
's%d%d' % (block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
'o%d' % block.output_filters,
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
......@@ -270,13 +316,15 @@ class BlockDecoder(object):
return block_strings
def efficientnet(width_coefficient=None,
depth_coefficient=None,
dropout_rate=0.2,
drop_connect_rate=0.2,
image_size=None,
num_classes=1000):
""" Get block arguments according to parameter and coefficients. """
def efficientnet(
width_coefficient=None,
depth_coefficient=None,
dropout_rate=0.2,
drop_connect_rate=0.2,
image_size=None,
num_classes=1000,
):
"""Get block arguments according to parameter and coefficients."""
blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25',
'r2_k3_s22_e6_i16_o24_se0.25',
......@@ -305,67 +353,51 @@ def efficientnet(width_coefficient=None,
def get_model_params(model_name, override_params):
""" Get the block args and global params for a given model """
"""Get the block args and global params for a given model"""
if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
blocks_args, global_params = efficientnet(width_coefficient=w,
depth_coefficient=d,
dropout_rate=p,
image_size=s)
blocks_args, global_params = efficientnet(
width_coefficient=w,
depth_coefficient=d,
dropout_rate=p,
image_size=s,
)
else:
raise NotImplementedError('model name is not pre-defined: %s' %
model_name)
raise NotImplementedError(
'model name is not pre-defined: %s' % model_name
)
if override_params:
global_params = global_params._replace(**override_params)
return blocks_args, global_params
url_map = {
'efficientnet-b0':
'/home/aistudio/data/weights/efficientnet-b0-355c32eb.pdparams',
'efficientnet-b1':
'/home/aistudio/data/weights/efficientnet-b1-f1951068.pdparams',
'efficientnet-b2':
'/home/aistudio/data/weights/efficientnet-b2-8bb594d6.pdparams',
'efficientnet-b3':
'/home/aistudio/data/weights/efficientnet-b3-5fb5a3c3.pdparams',
'efficientnet-b4':
'/home/aistudio/data/weights/efficientnet-b4-6ed6700e.pdparams',
'efficientnet-b5':
'/home/aistudio/data/weights/efficientnet-b5-b6417697.pdparams',
'efficientnet-b6':
'/home/aistudio/data/weights/efficientnet-b6-c76e70fd.pdparams',
'efficientnet-b7':
'/home/aistudio/data/weights/efficientnet-b7-dcc49843.pdparams',
'efficientnet-b0': '/home/aistudio/data/weights/efficientnet-b0-355c32eb.pdparams',
'efficientnet-b1': '/home/aistudio/data/weights/efficientnet-b1-f1951068.pdparams',
'efficientnet-b2': '/home/aistudio/data/weights/efficientnet-b2-8bb594d6.pdparams',
'efficientnet-b3': '/home/aistudio/data/weights/efficientnet-b3-5fb5a3c3.pdparams',
'efficientnet-b4': '/home/aistudio/data/weights/efficientnet-b4-6ed6700e.pdparams',
'efficientnet-b5': '/home/aistudio/data/weights/efficientnet-b5-b6417697.pdparams',
'efficientnet-b6': '/home/aistudio/data/weights/efficientnet-b6-c76e70fd.pdparams',
'efficientnet-b7': '/home/aistudio/data/weights/efficientnet-b7-dcc49843.pdparams',
}
url_map_advprop = {
'efficientnet-b0':
'/home/aistudio/data/weights/adv-efficientnet-b0-b64d5a18.pdparams',
'efficientnet-b1':
'/home/aistudio/data/weights/adv-efficientnet-b1-0f3ce85a.pdparams',
'efficientnet-b2':
'/home/aistudio/data/weights/adv-efficientnet-b2-6e9d97e5.pdparams',
'efficientnet-b3':
'/home/aistudio/data/weights/adv-efficientnet-b3-cdd7c0f4.pdparams',
'efficientnet-b4':
'/home/aistudio/data/weights/adv-efficientnet-b4-44fb3a87.pdparams',
'efficientnet-b5':
'/home/aistudio/data/weights/adv-efficientnet-b5-86493f6b.pdparams',
'efficientnet-b6':
'/home/aistudio/data/weights/adv-efficientnet-b6-ac80338e.pdparams',
'efficientnet-b7':
'/home/aistudio/data/weights/adv-efficientnet-b7-4652b6dd.pdparams',
'efficientnet-b8':
'/home/aistudio/data/weights/adv-efficientnet-b8-22a8fe65.pdparams',
'efficientnet-b0': '/home/aistudio/data/weights/adv-efficientnet-b0-b64d5a18.pdparams',
'efficientnet-b1': '/home/aistudio/data/weights/adv-efficientnet-b1-0f3ce85a.pdparams',
'efficientnet-b2': '/home/aistudio/data/weights/adv-efficientnet-b2-6e9d97e5.pdparams',
'efficientnet-b3': '/home/aistudio/data/weights/adv-efficientnet-b3-cdd7c0f4.pdparams',
'efficientnet-b4': '/home/aistudio/data/weights/adv-efficientnet-b4-44fb3a87.pdparams',
'efficientnet-b5': '/home/aistudio/data/weights/adv-efficientnet-b5-86493f6b.pdparams',
'efficientnet-b6': '/home/aistudio/data/weights/adv-efficientnet-b6-ac80338e.pdparams',
'efficientnet-b7': '/home/aistudio/data/weights/adv-efficientnet-b7-4652b6dd.pdparams',
'efficientnet-b8': '/home/aistudio/data/weights/adv-efficientnet-b8-22a8fe65.pdparams',
}
def load_pretrained_weights(model,
model_name,
weights_path=None,
load_fc=True,
advprop=False):
def load_pretrained_weights(
model, model_name, weights_path=None, load_fc=True, advprop=False
):
"""Loads pretrained weights from weights path or download using url.
Args:
model (Module): The whole model of efficientnet.
......
paddle/infrt/tests/models/linear.py
浏览文件 @ 7097630f
......@@ -28,13 +28,12 @@ CLASS_NUM = 10
# define a random dataset
class RandomDataset(paddle.io.Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples
def __getitem__(self, idx):
image = np.random.random([IMAGE_SIZE]).astype('float32')
label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype('int64')
return image, label
def __len__(self):
......@@ -42,7 +41,6 @@ class RandomDataset(paddle.io.Dataset):
class LinearNet(nn.Layer):
def __init__(self):
super(LinearNet, self).__init__()
self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
......@@ -71,11 +69,9 @@ adam = opt.Adam(learning_rate=0.001, parameters=layer.parameters())
# create data loader
dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
loader = paddle.io.DataLoader(dataset,
batch_size=BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=2)
loader = paddle.io.DataLoader(
dataset, batch_size=BATCH_SIZE, shuffle=True, drop_last=True, num_workers=2
)
# train
train(layer, loader, loss_fn, adam)
......
paddle/infrt/tests/models/resnet50_model.py
浏览文件 @ 7097630f
......@@ -19,6 +19,7 @@ from paddle.static import InputSpec
import sys
model = resnet50(True)
net = to_static(model,
input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')])
net = to_static(
model, input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')]
)
paddle.jit.save(net, sys.argv[1])
paddle/phi/api/yaml/generator/api_gen.py
浏览文件 @ 7097630f
......@@ -20,23 +20,24 @@ from api_base import BaseAPI, PREFIX_TENSOR_NAME
inplace_out_type_map = {
"Tensor": "Tensor&",
"std::vector<Tensor>": "std::vector<Tensor>&"
"std::vector<Tensor>": "std::vector<Tensor>&",
}
inplace_optional_out_type_map = {
"Tensor": "paddle::optional<Tensor>&",
"std::vector<Tensor>": "paddle::optional<std::vector<Tensor>>&"
"std::vector<Tensor>": "paddle::optional<std::vector<Tensor>>&",
}
class ForwardAPI(BaseAPI):
def __init__(self, api_item_yaml):
super(ForwardAPI, self).__init__(api_item_yaml)
self.is_dygraph_api, self.intermediate_outs = self.parse_intermediate(
api_item_yaml)
api_item_yaml
)
self.inplace_map, self.view_map = self.parse_inplace_and_view(
api_item_yaml)
api_item_yaml
)
def get_api_func_name(self):
if self.is_dygraph_api:
......@@ -47,17 +48,26 @@ class ForwardAPI(BaseAPI):
def gene_input(self, kernel_tensor_type=None, code_indent=''):
kernel_param = self.kernel['param']
input_name_tensor_map, input_tensor_code = super().gene_input(
kernel_tensor_type, code_indent)
kernel_tensor_type, code_indent
)
# generate the input that is in view list
for i, input_name in enumerate(self.inputs['names']):
if input_name in self.view_map.values(
) and input_name not in input_name_tensor_map.keys():
if kernel_tensor_type is None or kernel_tensor_type[0][
kernel_param.index(input_name)] == 'dense':
if (
input_name in self.view_map.values()
and input_name not in input_name_tensor_map.keys()
):
if (
kernel_tensor_type is None
or kernel_tensor_type[0][kernel_param.index(input_name)]
== 'dense'
):
trans_flag = self.gene_trans_flag(input_name)
input_tensor_code = input_tensor_code + f"""
input_tensor_code = (
input_tensor_code
+ f"""
{code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = PrepareData({input_name}, kernel.InputAt(0), {trans_flag});"""
)
else:
# do nothing
pass
......@@ -87,10 +97,12 @@ class ForwardAPI(BaseAPI):
result = re.search(r"(?P<in>\w+)\s*->\s*(?P<out>\w+)", item)
in_val = result.group('in')
out_val = result.group('out')
assert in_val in self.inputs['names'], \
f"{self.api} : {mode} input error: the input var name('{in_val}') is not found in the input args of {self.api}."
assert out_val in self.outputs['names'], \
f"{self.api} : {mode} output error: the output var name('{out_val}') is not found in the output args of {self.api}."
assert (
in_val in self.inputs['names']
), f"{self.api} : {mode} input error: the input var name('{in_val}') is not found in the input args of {self.api}."
assert (
out_val in self.outputs['names']
), f"{self.api} : {mode} output error: the output var name('{out_val}') is not found in the output args of {self.api}."
if mode == 'inplace':
inplace_map[out_val] = in_val
......@@ -106,7 +118,8 @@ class ForwardAPI(BaseAPI):
if inplace_flag and out_name in self.inplace_map:
if self.inplace_map[out_name] in self.optional_vars:
out_type_list.append(
inplace_optional_out_type_map[out_type])
inplace_optional_out_type_map[out_type]
)
else:
out_type_list.append(inplace_out_type_map[out_type])
else:
......@@ -124,7 +137,8 @@ class ForwardAPI(BaseAPI):
if inplace_flag and out_name in self.inplace_map:
if self.inplace_map[out_name] in self.optional_vars:
out_type_list.append(
inplace_optional_out_type_map[out_type])
inplace_optional_out_type_map[out_type]
)
else:
out_type_list.append(inplace_out_type_map[out_type])
elif self.is_dygraph_api or out_name not in self.intermediate_outs:
......@@ -151,11 +165,13 @@ class ForwardAPI(BaseAPI):
]
return 'return std::make_tuple(' + ", ".join(selected_code) + ');'
def gene_output(self,
out_dtype_list,
out_tensor_type_list=None,
code_indent='',
inplace_flag=False):
def gene_output(
self,
out_dtype_list,
out_tensor_type_list=None,
code_indent='',
inplace_flag=False,
):
kernel_output = []
output_names = []
output_create = ""
......@@ -164,29 +180,48 @@ class ForwardAPI(BaseAPI):
if len(out_dtype_list) == 1:
kernel_output.append('kernel_out')
output_names.append('kernel_out')
inplace_assign = " = " + self.inplace_map[
self.outputs['names'][0]] if inplace_flag and self.outputs[
'names'][0] in self.inplace_map else ""
inplace_assign = (
" = " + self.inplace_map[self.outputs['names'][0]]
if inplace_flag and self.outputs['names'][0] in self.inplace_map
else ""
)
output_create = f"""
{code_indent} {return_type} api_output{inplace_assign};"""
set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[
0] == 'dense' else 'SetSelectedRowsKernelOutput'
set_out_func = (
'SetKernelOutput'
if out_tensor_type_list is None
or out_tensor_type_list[0] == 'dense'
else 'SetSelectedRowsKernelOutput'
)
if return_type == 'std::vector<Tensor>':
assert self.outputs['out_size_expr'][0] is not None, \
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = output_create + f"""
assert (
self.outputs['out_size_expr'][0] is not None
), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out = {set_out_func}({self.outputs['out_size_expr'][0]}, &api_output);"""
)
else:
output_create = output_create + f"""
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out = {set_out_func}(&api_output);"""
if not inplace_flag and self.view_map is not None and self.outputs[
'names'][0] in self.view_map:
output_create = output_create + f"""
)
if (
not inplace_flag
and self.view_map is not None
and self.outputs['names'][0] in self.view_map
):
output_create = (
output_create
+ f"""
{code_indent} kernel_out->ShareBufferWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][0]]});
{code_indent} kernel_out->ShareInplaceVersionCounterWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][0]]});
{code_indent} VLOG(3) << "Perform View between Output and Input Tensor, share allocation and inplace version.";"""
)
elif len(out_dtype_list) > 1:
output_create = f"""
......@@ -206,47 +241,74 @@ class ForwardAPI(BaseAPI):
for i in range(len(out_dtype_list)):
kernel_output.append(f'kernel_out_{i}')
output_names.append(f'kernel_out_{i}')
set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[
i] == 'dense' else 'SetSelectedRowsKernelOutput'
set_out_func = (
'SetKernelOutput'
if out_tensor_type_list is None
or out_tensor_type_list[i] == 'dense'
else 'SetSelectedRowsKernelOutput'
)
get_out_code = f"&std::get<{i}>(api_output)"
if self.outputs['names'][
i] in self.inplace_map and self.inplace_map[
self.outputs['names'][i]] in self.optional_vars:
if (
self.outputs['names'][i] in self.inplace_map
and self.inplace_map[self.outputs['names'][i]]
in self.optional_vars
):
get_out_code = f"std::get<{i}>(api_output).get_ptr()"
if out_dtype_list[i] == 'std::vector<Tensor>':
assert self.outputs['out_size_expr'][i] is not None, \
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
assert (
self.outputs['out_size_expr'][i] is not None
), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
# Special case for inplace vector and inplace optional<vector>
if self.outputs['names'][i] in self.inplace_map:
set_out_func = "SetInplaceVectorKernelOutput"
if self.inplace_map[self.outputs['names']
[i]] in self.optional_vars:
set_out_func = "SetInplaceOptionalVectorKernelOutput"
if (
self.inplace_map[self.outputs['names'][i]]
in self.optional_vars
):
set_out_func = (
"SetInplaceOptionalVectorKernelOutput"
)
get_out_code = f"std::get<{i}>(api_output)"
output_create = output_create + f"""
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out_{i} = {set_out_func}({self.outputs['out_size_expr'][i]}, {get_out_code});"""
)
else:
output_create = output_create + f"""
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out_{i} = {set_out_func}({get_out_code});"""
)
if not inplace_flag and self.view_map is not None and self.outputs[
'names'][i] in self.view_map:
if (
not inplace_flag
and self.view_map is not None
and self.outputs['names'][i] in self.view_map
):
if out_dtype_list[i] == 'Tensor':
output_create = output_create + f"""
output_create = (
output_create
+ f"""
{code_indent} kernel_out_{i}->ShareBufferWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][i]]});
{code_indent} kernel_out_{i}->ShareInplaceVersionCounterWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][i]]});
{code_indent} VLOG(3) << "Perform View between Output and Input Tensor, share allocation and inplace version.";"""
)
else:
raise ValueError(
"{} : Output error: only support Tensor type when use view in yaml. But get {}"
.format(self.api, out_dtype_list[i]))
"{} : Output error: only support Tensor type when use view in yaml. But get {}".format(
self.api, out_dtype_list[i]
)
)
else:
raise ValueError(
"{} : Output error: the output should not be empty.".format(
self.api))
self.api
)
)
return kernel_output, output_names, output_create
......@@ -288,15 +350,18 @@ DECLARE_bool(conv2d_disable_cudnn);
def api_namespace():
return ("""
return (
"""
namespace paddle {
namespace experimental {
""", """
""",
"""
} // namespace experimental
} // namespace paddle
""")
""",
)
def generate_api(api_yaml_path, header_file_path, source_file_path):
......@@ -338,19 +403,26 @@ def generate_api(api_yaml_path, header_file_path, source_file_path):
def main():
parser = argparse.ArgumentParser(
description='Generate PaddlePaddle C++ API files')
parser.add_argument('--api_yaml_path',
help='path to api yaml file',
nargs='+',
default='paddle/phi/api/yaml/ops.yaml')
parser.add_argument('--api_header_path',
help='output of generated api header code file',
default='paddle/phi/api/include/api.h')
parser.add_argument('--api_source_path',
help='output of generated api source code file',
default='paddle/phi/api/lib/api.cc')
description='Generate PaddlePaddle C++ API files'
)
parser.add_argument(
'--api_yaml_path',
help='path to api yaml file',
nargs='+',
default='paddle/phi/api/yaml/ops.yaml',
)
parser.add_argument(
'--api_header_path',
help='output of generated api header code file',
default='paddle/phi/api/include/api.h',
)
parser.add_argument(
'--api_source_path',
help='output of generated api source code file',
default='paddle/phi/api/lib/api.cc',
)
options = parser.parse_args()
......
paddle/phi/api/yaml/generator/backward_api_gen.py
浏览文件 @ 7097630f
......@@ -20,7 +20,6 @@ from api_base import BaseAPI
class BackwardAPI(BaseAPI):
def __init__(self, backward_item_yaml):
super(BackwardAPI, self).__init__(backward_item_yaml)
self.check_args(backward_item_yaml['forward'])
......@@ -33,12 +32,18 @@ class BackwardAPI(BaseAPI):
# api_name (const Tensor& input, ... , int attr, ...) -> Tensor(out)
result = re.search(
r"(?P<op>[a-z][a-z0-9_]+)\s*(?P<args>\([^\)]+\))\s*->\s*(?P<outputs>.+)",
forward_config)
forward_config,
)
api = result.group('op')
_, outputs, _, = self.parse_output(self.api, result.group('outputs'))
(
_,
outputs,
_,
) = self.parse_output(self.api, result.group('outputs'))
outputs = [item.split('@')[0] for item in outputs]
fw_inputs, fw_attrs = self.parse_input_and_attr(api,
result.group('args'))
fw_inputs, fw_attrs = self.parse_input_and_attr(
api, result.group('args')
)
return api, fw_inputs, fw_attrs, outputs
......@@ -54,27 +59,34 @@ class BackwardAPI(BaseAPI):
def check_args(self, forward_config):
# parse the forward and backward config
_, fw_inputs, fw_attrs, fw_outputs = self.parse_forward_config(
forward_config)
forward_config
)
# check the inputs of backward
for input in self.inputs['names']:
if input not in fw_inputs['names'] and input not in fw_outputs:
if input.endswith('_grad'):
original_name = input[:-5]
assert original_name in fw_outputs, \
f"{self.api} : Input Tensor error: the input tensor({input}) of backward should be an input or output or grad of output in forward api. \
assert (
original_name in fw_outputs
), f"{self.api} : Input Tensor error: the input tensor({input}) of backward should be an input or output or grad of output in forward api. \
Please check the forward of {self.api} in yaml."
# check the attributes of backward
for attr in self.attrs['names']:
assert (attr in fw_attrs['names'] and self.attrs['attr_info'][attr][0] == fw_attrs['attr_info'][attr][0]) or \
self.attrs['attr_info'][attr][1] is not None, \
f"{self.api} : Attribute error: The attribute({attr}) of backward isn't consistent with forward api or doesn't have default value. \
assert (
attr in fw_attrs['names']
and self.attrs['attr_info'][attr][0]
== fw_attrs['attr_info'][attr][0]
) or self.attrs['attr_info'][attr][
1
] is not None, f"{self.api} : Attribute error: The attribute({attr}) of backward isn't consistent with forward api or doesn't have default value. \
Please check the args of {self.api} in yaml."
# check the output of backward
assert len(self.outputs['types']) <= len(fw_inputs['names']), \
f"{self.api} : Output error: The number of outputs should be less then the number of inputs of forward api. \
assert len(self.outputs['types']) <= len(
fw_inputs['names']
), f"{self.api} : Output error: The number of outputs should be less then the number of inputs of forward api. \
Please check the output of {self.api} in yaml."
def get_declare_args(self, inplace_flag=False):
......@@ -83,13 +95,16 @@ class BackwardAPI(BaseAPI):
def get_define_args(self, inplace_flag=False):
out_type_map = {
'Tensor': 'Tensor*',
'std::vector<Tensor>': 'std::vector<Tensor*>'
'std::vector<Tensor>': 'std::vector<Tensor*>',
}
intputs_and_attrs = super(BackwardAPI, self).get_define_args()
outs = []
for i, name in enumerate(self.outputs['names']):
outs.append(out_type_map[self.outputs['types'][i]] + ' ' +
name.split('@')[0])
outs.append(
out_type_map[self.outputs['types'][i]]
+ ' '
+ name.split('@')[0]
)
result = intputs_and_attrs + ', ' + ", ".join(outs)
return result
......@@ -100,7 +115,8 @@ class BackwardAPI(BaseAPI):
if not self.is_base_api:
invoke_func_name = self.invoke.split('(')[0]
if (not invoke_func_name.endswith("_grad")) and (
not invoke_func_name.endswith('_impl')):
not invoke_func_name.endswith('_impl')
):
return ""
api_func_name = self.get_api_func_name()
api_declaration = f"""
......@@ -124,11 +140,13 @@ PADDLE_API void {api_func_name}({self.get_declare_args()});
def get_return_type(self, inplace_flag=False):
return 'void'
def gene_output(self,
out_dtype_list,
out_tensor_type_list=None,
code_indent='',
inplace_flag=False):
def gene_output(
self,
out_dtype_list,
out_tensor_type_list=None,
code_indent='',
inplace_flag=False,
):
kernel_output = []
output_names = []
output_create = ""
......@@ -136,60 +154,101 @@ PADDLE_API void {api_func_name}({self.get_declare_args()});
if len(out_dtype_list) == 1:
kernel_output.append('kernel_out')
output_names.append('kernel_out')
inplace_assign = " = " + self.inplace_map[self.outputs['names'][
0]] if inplace_flag and self.inplace_map is not None and self.outputs[
'names'][0] in self.inplace_map else ""
inplace_assign = (
" = " + self.inplace_map[self.outputs['names'][0]]
if inplace_flag
and self.inplace_map is not None
and self.outputs['names'][0] in self.inplace_map
else ""
)
output_create = ""
set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[
0] == 'dense' else 'SetSelectedRowsKernelOutput'
set_out_func = (
'SetKernelOutput'
if out_tensor_type_list is None
or out_tensor_type_list[0] == 'dense'
else 'SetSelectedRowsKernelOutput'
)
if out_dtype_list[0] == 'std::vector<Tensor>':
assert self.outputs['out_size_expr'] is not None, \
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = output_create + f"""
assert (
self.outputs['out_size_expr'] is not None
), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out = {set_out_func}(&{self.outputs['names'][0]});"""
)
else:
output_create = output_create + f"""
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out = {set_out_func}({self.outputs['names'][0]});"""
)
elif len(out_dtype_list) > 1:
output_create = ""
for i, out_type_item in enumerate(out_dtype_list):
kernel_output.append(f'kernel_out_{i}')
output_names.append(f'kernel_out_{i}')
set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[
i] == 'dense' else 'SetSelectedRowsKernelOutput'
set_out_func = (
'SetKernelOutput'
if out_tensor_type_list is None
or out_tensor_type_list[i] == 'dense'
else 'SetSelectedRowsKernelOutput'
)
if out_type_item == 'Tensor':
if inplace_flag and self.inplace_map is not None and self.outputs[
'names'][i] in self.inplace_map:
output_create = output_create + f"""
if (
inplace_flag
and self.inplace_map is not None
and self.outputs['names'][i] in self.inplace_map
):
output_create = (
output_create
+ f"""
{code_indent} *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};"""
)
output_create = output_create + f"""
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out_{i} = {set_out_func}({self.outputs['names'][i]});"""
)
else:
if inplace_flag and self.inplace_map is not None and self.outputs[
'names'][i] in self.inplace_map:
output_create = output_create + f"""
if (
inplace_flag
and self.inplace_map is not None
and self.outputs['names'][i] in self.inplace_map
):
output_create = (
output_create
+ f"""
{code_indent} *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};"""
assert self.outputs['out_size_expr'][i] is not None, \
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = output_create + f"""
)
assert (
self.outputs['out_size_expr'][i] is not None
), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = (
output_create
+ f"""
{code_indent} auto kernel_out_{i} = {set_out_func}(&{self.outputs['names'][i]});"""
)
else:
raise ValueError(
"{} : Output error: the output should not be empty.".format(
self.api))
self.api
)
)
return kernel_output, output_names, output_create
def gene_invoke_code(self, invoke_code, params_code):
invoke_func_name = invoke_code.split('(')[0].strip()
if invoke_func_name.endswith('_grad') or invoke_func_name.endswith(
'_impl'):
'_impl'
):
return f"""
PADDLE_API {self.get_return_type()} {self.api}({params_code}) {{
{invoke_code};
......@@ -235,19 +294,23 @@ DECLARE_bool(conv2d_disable_cudnn);
def backward_api_namespace():
return ("""
return (
"""
namespace paddle {
namespace experimental {
""", """
""",
"""
} // namespace experimental
} // namespace paddle
""")
""",
)
def generate_backward_api(backward_yaml_path, header_file_path,
source_file_path):
def generate_backward_api(
backward_yaml_path, header_file_path, source_file_path
):
bw_apis = []
for each_api_yaml in backward_yaml_path:
......@@ -283,18 +346,25 @@ def generate_backward_api(backward_yaml_path, header_file_path,
def main():
parser = argparse.ArgumentParser(
description='Generate PaddlePaddle C++ backward API files')
parser.add_argument('--backward_yaml_path',
help='path to backward yaml file',
nargs='+',
default='paddle/phi/api/yaml/backward.yaml')
parser.add_argument('--backward_header_path',
help='output of generated backward header code file',
default='paddle/phi/api/backward/backward_api.h')
parser.add_argument('--backward_source_path',
help='output of generated backward source code file',
default='paddle/phi/api/lib/backward_api.cc')
description='Generate PaddlePaddle C++ backward API files'
)
parser.add_argument(
'--backward_yaml_path',
help='path to backward yaml file',
nargs='+',
default='paddle/phi/api/yaml/backward.yaml',
)
parser.add_argument(
'--backward_header_path',
help='output of generated backward header code file',
default='paddle/phi/api/backward/backward_api.h',
)
parser.add_argument(
'--backward_source_path',
help='output of generated backward source code file',
default='paddle/phi/api/lib/backward_api.cc',
)
options = parser.parse_args()
......@@ -302,8 +372,9 @@ def main():
header_file_path = options.backward_header_path
source_file_path = options.backward_source_path
generate_backward_api(backward_yaml_path, header_file_path,
source_file_path)
generate_backward_api(
backward_yaml_path, header_file_path, source_file_path
)
if __name__ == '__main__':
......
paddle/phi/api/yaml/generator/cross_validate.py
浏览文件 @ 7097630f
......@@ -34,17 +34,22 @@ def main(forward_api_yaml_paths, backward_api_yaml_paths):
if __name__ == "__main__":
current_dir = Path(__file__).parent / "temp"
parser = argparse.ArgumentParser(
description="Parse api yaml into canonical format.")
parser.add_argument('--forward_yaml_paths',
type=str,
nargs='+',
default=str(current_dir / "api.parsed.yaml"),
help="forward api yaml file.")
parser.add_argument('--backward_yaml_paths',
type=str,
nargs='+',
default=str(current_dir / "backward_api.parsed.yaml"),
help="backward api yaml file.")
description="Parse api yaml into canonical format."
)
parser.add_argument(
'--forward_yaml_paths',
type=str,
nargs='+',
default=str(current_dir / "api.parsed.yaml"),
help="forward api yaml file.",
)
parser.add_argument(
'--backward_yaml_paths',
type=str,
nargs='+',
default=str(current_dir / "backward_api.parsed.yaml"),
help="backward api yaml file.",
)
args = parser.parse_args()
main(args.forward_yaml_paths, args.backward_yaml_paths)
paddle/phi/api/yaml/generator/filters.py
浏览文件 @ 7097630f
......@@ -15,12 +15,20 @@
import itertools
import re
from type_mapping import (input_types_map, optional_input_types_map,
attr_types_map, opmaker_attr_types_map,
output_type_map)
from type_mapping import (dense_input_types_map, dense_optional_input_types_map,
dense_output_types_map, sr_output_types_map,
phi_attr_types_map)
from type_mapping import (
input_types_map,
optional_input_types_map,
attr_types_map,
opmaker_attr_types_map,
output_type_map,
)
from type_mapping import (
dense_input_types_map,
dense_optional_input_types_map,
dense_output_types_map,
sr_output_types_map,
phi_attr_types_map,
)
def quote(s):
......@@ -104,7 +112,7 @@ def to_input_name(s):
is more common.
"""
match = re.match(r"(d\d*)(\w+)", s)
assert (match.group(1) != ""), "it should be a grad style name."
assert match.group(1) != "", "it should be a grad style name."
return match.group(2)
......@@ -117,9 +125,10 @@ def cartesian_prod_attrs(attrs):
items.append((name, "{}Tensor".format(name)))
elif type_name == "IntArray":
items.append(
(name, "{}Tensor".format(name), "{}TensorList".format(name)))
(name, "{}Tensor".format(name), "{}TensorList".format(name))
)
else:
items.append((name, ))
items.append((name,))
_combinations = itertools.product(*items)
combinations = []
......
paddle/scripts/paddle_build.sh
浏览文件 @ 7097630f
此差异已折叠。
pyproject.toml 0 → 100644
浏览文件 @ 7097630f
[tool.black]
exclude = "build"
line-length = 80
skip-string-normalization = true
python/paddle/__init__.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/amp/auto_cast.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/amp/grad_scaler.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/audio/__init__.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/audio/utils/error.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/autograd/__init__.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/autograd/py_layer.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/batch.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/callbacks.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/check_import_scipy.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/common_ops_import.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/cifar.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/common.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/conll05.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/flowers.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/image.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/imdb.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/imikolov.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/mnist.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/movielens.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/voc2012.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/wmt14.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/dataset/wmt16.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/device/__init__.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/device/cuda/graphs.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/distributed/spawn.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/distribution/beta.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/distribution/kl.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fft.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/__init__.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/average.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/backward.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/clip.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/communicator.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/compiler.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/core.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/data.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/data_feeder.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/dataset.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/debugger.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/dygraph/base.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/dygraph/io.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/dygraph/jit.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/dygraph/nn.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/dygraph/rnn.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/entry_attr.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/evaluator.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/executor.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/framework.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/generator.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/graphviz.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/initializer.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/input.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/io.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/ir.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/layer_helper.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/layers/io.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/layers/loss.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/layers/nn.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/layers/ops.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/layers/rnn.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/layers/utils.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/lazy_init.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/lod_tensor.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/metrics.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/net_drawer.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/nets.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/op.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/optimizer.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/param_attr.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/profiler.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/reader.py
浏览文件 @ 7097630f
此差异已折叠。
python/paddle/fluid/regularizer.py
浏览文件 @ 7097630f
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或