Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Greenplum
Pytorch Widedeep
提交
5d03f95e
P
Pytorch Widedeep
项目概览
Greenplum
/
Pytorch Widedeep
大约 1 年 前同步成功
通知
9
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Pytorch Widedeep
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
5d03f95e
编写于
9月 05, 2020
作者:
J
jrzaurin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
updated to new version of black and added quite to isort to avoid simlink warnings
上级
905e4d75
变更
7
显示空白变更内容
内联
并排
Showing
7 changed file
with
27 addition
and
42 deletion
+27
-42
code_style.sh
code_style.sh
+1
-1
examples/adult_script.py
examples/adult_script.py
+3
-1
pytorch_widedeep/models/deep_image.py
pytorch_widedeep/models/deep_image.py
+1
-2
pytorch_widedeep/models/wide_deep.py
pytorch_widedeep/models/wide_deep.py
+1
-1
pytorch_widedeep/preprocessing/_preprocessors.py
pytorch_widedeep/preprocessing/_preprocessors.py
+14
-23
pytorch_widedeep/utils/dense_utils.py
pytorch_widedeep/utils/dense_utils.py
+3
-6
pytorch_widedeep/utils/fastai_transforms.py
pytorch_widedeep/utils/fastai_transforms.py
+4
-8
未找到文件。
code_style.sh
浏览文件 @
5d03f95e
# sort imports
# sort imports
isort
.
pytorch_widedeep tests examples setup.py
isort
--quiet
.
pytorch_widedeep tests examples setup.py
# Black code style
# Black code style
black
.
pytorch_widedeep tests examples setup.py
black
.
pytorch_widedeep tests examples setup.py
# flake8 standards
# flake8 standards
...
...
examples/adult_script.py
浏览文件 @
5d03f95e
...
@@ -104,4 +104,6 @@ if __name__ == "__main__":
...
@@ -104,4 +104,6 @@ if __name__ == "__main__":
# model = WideDeep(wide=wide, deepdense=deepdense)
# model = WideDeep(wide=wide, deepdense=deepdense)
# model.load_state_dict(torch.load("model_weights/model_dict.t"))
# model.load_state_dict(torch.load("model_weights/model_dict.t"))
# # <All keys matched successfully>
# # <All keys matched successfully>
import
pdb
;
pdb
.
set_trace
()
# breakpoint dde47114 //
import
pdb
pdb
.
set_trace
()
# breakpoint dde47114 //
pytorch_widedeep/models/deep_image.py
浏览文件 @
5d03f95e
...
@@ -166,8 +166,7 @@ class DeepImage(nn.Module):
...
@@ -166,8 +166,7 @@ class DeepImage(nn.Module):
self
.
output_dim
=
head_layers
[
-
1
]
self
.
output_dim
=
head_layers
[
-
1
]
def
forward
(
self
,
x
:
Tensor
)
->
Tensor
:
# type: ignore
def
forward
(
self
,
x
:
Tensor
)
->
Tensor
:
# type: ignore
r
"""Forward pass connecting the `'backbone'` with the `'head layers'`
r
"""Forward pass connecting the `'backbone'` with the `'head layers'`"""
"""
x
=
self
.
backbone
(
x
)
x
=
self
.
backbone
(
x
)
x
=
x
.
view
(
x
.
size
(
0
),
-
1
)
x
=
x
.
view
(
x
.
size
(
0
),
-
1
)
if
self
.
head_layers
is
not
None
:
if
self
.
head_layers
is
not
None
:
...
...
pytorch_widedeep/models/wide_deep.py
浏览文件 @
5d03f95e
pytorch_widedeep/preprocessing/_preprocessors.py
浏览文件 @
5d03f95e
...
@@ -93,15 +93,16 @@ class WidePreprocessor(BasePreprocessor):
...
@@ -93,15 +93,16 @@ class WidePreprocessor(BasePreprocessor):
"""
"""
def
__init__
(
def
__init__
(
self
,
wide_cols
:
List
[
str
],
crossed_cols
=
None
,
self
,
wide_cols
:
List
[
str
],
crossed_cols
=
None
,
):
):
super
(
WidePreprocessor
,
self
).
__init__
()
super
(
WidePreprocessor
,
self
).
__init__
()
self
.
wide_cols
=
wide_cols
self
.
wide_cols
=
wide_cols
self
.
crossed_cols
=
crossed_cols
self
.
crossed_cols
=
crossed_cols
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
BasePreprocessor
:
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
BasePreprocessor
:
"""Fits the Preprocessor and creates required attributes
"""Fits the Preprocessor and creates required attributes"""
"""
df_wide
=
self
.
_prepare_wide
(
df
)
df_wide
=
self
.
_prepare_wide
(
df
)
self
.
wide_crossed_cols
=
df_wide
.
columns
.
tolist
()
self
.
wide_crossed_cols
=
df_wide
.
columns
.
tolist
()
vocab
=
self
.
_make_global_feature_list
(
df_wide
[
self
.
wide_crossed_cols
])
vocab
=
self
.
_make_global_feature_list
(
df_wide
[
self
.
wide_crossed_cols
])
...
@@ -110,8 +111,7 @@ class WidePreprocessor(BasePreprocessor):
...
@@ -110,8 +111,7 @@ class WidePreprocessor(BasePreprocessor):
return
self
return
self
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
array
:
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
array
:
r
"""Returns the processed dataframe
r
"""Returns the processed dataframe"""
"""
try
:
try
:
self
.
feature_dict
self
.
feature_dict
except
:
except
:
...
@@ -147,8 +147,7 @@ class WidePreprocessor(BasePreprocessor):
...
@@ -147,8 +147,7 @@ class WidePreprocessor(BasePreprocessor):
return
decoded
return
decoded
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
"""Combines ``fit`` and ``transform``
"""Combines ``fit`` and ``transform``"""
"""
return
self
.
fit
(
df
).
transform
(
df
)
return
self
.
fit
(
df
).
transform
(
df
)
def
_make_global_feature_list
(
self
,
df
:
pd
.
DataFrame
)
->
List
:
def
_make_global_feature_list
(
self
,
df
:
pd
.
DataFrame
)
->
List
:
...
@@ -256,8 +255,7 @@ class DensePreprocessor(BasePreprocessor):
...
@@ -256,8 +255,7 @@ class DensePreprocessor(BasePreprocessor):
),
"'embed_cols' and 'continuous_cols' are 'None'. Please, define at least one of the two."
),
"'embed_cols' and 'continuous_cols' are 'None'. Please, define at least one of the two."
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
BasePreprocessor
:
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
BasePreprocessor
:
"""Fits the Preprocessor and creates required attributes
"""Fits the Preprocessor and creates required attributes"""
"""
if
self
.
embed_cols
is
not
None
:
if
self
.
embed_cols
is
not
None
:
df_emb
=
self
.
_prepare_embed
(
df
)
df_emb
=
self
.
_prepare_embed
(
df
)
self
.
label_encoder
=
LabelEncoder
(
df_emb
.
columns
.
tolist
()).
fit
(
df_emb
)
self
.
label_encoder
=
LabelEncoder
(
df_emb
.
columns
.
tolist
()).
fit
(
df_emb
)
...
@@ -274,8 +272,7 @@ class DensePreprocessor(BasePreprocessor):
...
@@ -274,8 +272,7 @@ class DensePreprocessor(BasePreprocessor):
return
self
return
self
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
"""Returns the processed ``dataframe`` as a np.ndarray
"""Returns the processed ``dataframe`` as a np.ndarray"""
"""
if
self
.
embed_cols
is
not
None
:
if
self
.
embed_cols
is
not
None
:
df_emb
=
self
.
_prepare_embed
(
df
)
df_emb
=
self
.
_prepare_embed
(
df
)
df_emb
=
self
.
label_encoder
.
transform
(
df_emb
)
df_emb
=
self
.
label_encoder
.
transform
(
df_emb
)
...
@@ -302,8 +299,7 @@ class DensePreprocessor(BasePreprocessor):
...
@@ -302,8 +299,7 @@ class DensePreprocessor(BasePreprocessor):
return
df_deep
.
values
return
df_deep
.
values
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
"""Combines ``fit`` and ``transform``
"""Combines ``fit`` and ``transform``"""
"""
return
self
.
fit
(
df
).
transform
(
df
)
return
self
.
fit
(
df
).
transform
(
df
)
def
_prepare_embed
(
self
,
df
:
pd
.
DataFrame
)
->
pd
.
DataFrame
:
def
_prepare_embed
(
self
,
df
:
pd
.
DataFrame
)
->
pd
.
DataFrame
:
...
@@ -387,8 +383,7 @@ class TextPreprocessor(BasePreprocessor):
...
@@ -387,8 +383,7 @@ class TextPreprocessor(BasePreprocessor):
self
.
verbose
=
verbose
self
.
verbose
=
verbose
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
BasePreprocessor
:
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
BasePreprocessor
:
"""Builds the vocabulary
"""Builds the vocabulary"""
"""
texts
=
df
[
self
.
text_col
].
tolist
()
texts
=
df
[
self
.
text_col
].
tolist
()
tokens
=
get_texts
(
texts
)
tokens
=
get_texts
(
texts
)
self
.
vocab
=
Vocab
.
create
(
self
.
vocab
=
Vocab
.
create
(
...
@@ -399,8 +394,7 @@ class TextPreprocessor(BasePreprocessor):
...
@@ -399,8 +394,7 @@ class TextPreprocessor(BasePreprocessor):
return
self
return
self
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
"""Returns the padded, `numericalised` sequences
"""Returns the padded, `numericalised` sequences"""
"""
try
:
try
:
self
.
vocab
self
.
vocab
except
:
except
:
...
@@ -419,8 +413,7 @@ class TextPreprocessor(BasePreprocessor):
...
@@ -419,8 +413,7 @@ class TextPreprocessor(BasePreprocessor):
return
padded_seq
return
padded_seq
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
"""Combines ``fit`` and ``transform``
"""Combines ``fit`` and ``transform``"""
"""
return
self
.
fit
(
df
).
transform
(
df
)
return
self
.
fit
(
df
).
transform
(
df
)
...
@@ -502,8 +495,7 @@ class ImagePreprocessor(BasePreprocessor):
...
@@ -502,8 +495,7 @@ class ImagePreprocessor(BasePreprocessor):
return
self
return
self
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
"""Resizes the images to the input height and width.
"""Resizes the images to the input height and width."""
"""
try
:
try
:
self
.
aap
self
.
aap
except
:
except
:
...
@@ -564,6 +556,5 @@ class ImagePreprocessor(BasePreprocessor):
...
@@ -564,6 +556,5 @@ class ImagePreprocessor(BasePreprocessor):
return
np
.
asarray
(
resized_imgs
)
return
np
.
asarray
(
resized_imgs
)
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
def
fit_transform
(
self
,
df
:
pd
.
DataFrame
)
->
np
.
ndarray
:
"""Combines ``fit`` and ``transform``
"""Combines ``fit`` and ``transform``"""
"""
return
self
.
fit
(
df
).
transform
(
df
)
return
self
.
fit
(
df
).
transform
(
df
)
pytorch_widedeep/utils/dense_utils.py
浏览文件 @
5d03f95e
...
@@ -45,8 +45,7 @@ class LabelEncoder(object):
...
@@ -45,8 +45,7 @@ class LabelEncoder(object):
self
.
columns_to_encode
=
columns_to_encode
self
.
columns_to_encode
=
columns_to_encode
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
"LabelEncoder"
:
def
fit
(
self
,
df
:
pd
.
DataFrame
)
->
"LabelEncoder"
:
"""Creates encoding attributes
"""Creates encoding attributes"""
"""
df_inp
=
df
.
copy
()
df_inp
=
df
.
copy
()
...
@@ -78,8 +77,7 @@ class LabelEncoder(object):
...
@@ -78,8 +77,7 @@ class LabelEncoder(object):
return
self
return
self
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
pd
.
DataFrame
:
def
transform
(
self
,
df
:
pd
.
DataFrame
)
->
pd
.
DataFrame
:
"""Label Encoded the categories in ``columns_to_encode``
"""Label Encoded the categories in ``columns_to_encode``"""
"""
try
:
try
:
self
.
encoding_dict
self
.
encoding_dict
except
AttributeError
:
except
AttributeError
:
...
@@ -126,8 +124,7 @@ class LabelEncoder(object):
...
@@ -126,8 +124,7 @@ class LabelEncoder(object):
return
self
.
fit
(
df
).
transform
(
df
)
return
self
.
fit
(
df
).
transform
(
df
)
def
inverse_transform
(
self
,
df
:
pd
.
DataFrame
)
->
pd
.
DataFrame
:
def
inverse_transform
(
self
,
df
:
pd
.
DataFrame
)
->
pd
.
DataFrame
:
"""Returns the original categories
"""Returns the original categories"""
"""
for
k
,
v
in
self
.
inverse_encoding_dict
.
items
():
for
k
,
v
in
self
.
inverse_encoding_dict
.
items
():
df
[
k
]
=
df
[
k
].
apply
(
lambda
x
:
v
[
x
])
df
[
k
]
=
df
[
k
].
apply
(
lambda
x
:
v
[
x
])
...
...
pytorch_widedeep/utils/fastai_transforms.py
浏览文件 @
5d03f95e
...
@@ -78,8 +78,7 @@ defaults.text_spec_tok = [UNK, PAD, BOS, EOS, FLD, TK_MAJ, TK_UP, TK_REP, TK_WRE
...
@@ -78,8 +78,7 @@ defaults.text_spec_tok = [UNK, PAD, BOS, EOS, FLD, TK_MAJ, TK_UP, TK_REP, TK_WRE
class
BaseTokenizer
:
class
BaseTokenizer
:
"""Basic class for a tokenizer function.
"""Basic class for a tokenizer function."""
"""
def
__init__
(
self
,
lang
:
str
):
def
__init__
(
self
,
lang
:
str
):
self
.
lang
=
lang
self
.
lang
=
lang
...
@@ -278,8 +277,7 @@ class Tokenizer:
...
@@ -278,8 +277,7 @@ class Tokenizer:
return
toks
return
toks
def
_process_all_1
(
self
,
texts
:
Collection
[
str
])
->
List
[
List
[
str
]]:
def
_process_all_1
(
self
,
texts
:
Collection
[
str
])
->
List
[
List
[
str
]]:
"""Process a list of ``texts`` in one process.
"""Process a list of ``texts`` in one process."""
"""
tok
=
self
.
tok_func
(
self
.
lang
)
tok
=
self
.
tok_func
(
self
.
lang
)
if
self
.
special_cases
:
if
self
.
special_cases
:
...
@@ -332,13 +330,11 @@ class Vocab:
...
@@ -332,13 +330,11 @@ class Vocab:
self
.
stoi
=
defaultdict
(
int
,
{
v
:
k
for
k
,
v
in
enumerate
(
self
.
itos
)})
self
.
stoi
=
defaultdict
(
int
,
{
v
:
k
for
k
,
v
in
enumerate
(
self
.
itos
)})
def
numericalize
(
self
,
t
:
Collection
[
str
])
->
List
[
int
]:
def
numericalize
(
self
,
t
:
Collection
[
str
])
->
List
[
int
]:
"""Convert a list of str (or tokens) ``t`` to their ids.
"""Convert a list of str (or tokens) ``t`` to their ids."""
"""
return
[
self
.
stoi
[
w
]
for
w
in
t
]
return
[
self
.
stoi
[
w
]
for
w
in
t
]
def
textify
(
self
,
nums
:
Collection
[
int
],
sep
=
" "
)
->
List
[
str
]:
def
textify
(
self
,
nums
:
Collection
[
int
],
sep
=
" "
)
->
List
[
str
]:
"""Convert a list of ``nums`` (or indexes) to their tokens.
"""Convert a list of ``nums`` (or indexes) to their tokens."""
"""
return
sep
.
join
([
self
.
itos
[
i
]
for
i
in
nums
])
if
sep
is
not
None
else
[
self
.
itos
[
i
]
for
i
in
nums
]
# type: ignore
return
sep
.
join
([
self
.
itos
[
i
]
for
i
in
nums
])
if
sep
is
not
None
else
[
self
.
itos
[
i
]
for
i
in
nums
]
# type: ignore
def
__getstate__
(
self
):
def
__getstate__
(
self
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录