Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
曾经的那一瞬间
Models
提交
ce8f9010
M
Models
项目概览
曾经的那一瞬间
/
Models
11 个月 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
Models
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ce8f9010
编写于
5月 27, 2023
作者:
A
A. Unique TensorFlower
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Internal change
PiperOrigin-RevId: 535815173
上级
799abc1c
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
25 addition
and
7 deletion
+25
-7
official/nlp/modeling/networks/funnel_transformer.py
official/nlp/modeling/networks/funnel_transformer.py
+18
-5
official/nlp/modeling/networks/funnel_transformer_test.py
official/nlp/modeling/networks/funnel_transformer_test.py
+7
-2
未找到文件。
official/nlp/modeling/networks/funnel_transformer.py
浏览文件 @
ce8f9010
...
...
@@ -298,6 +298,7 @@ class FunnelTransformerEncoder(tf.keras.layers.Layer):
str
,
tf
.
keras
.
layers
.
Layer
]
=
layers
.
TransformerEncoderBlock
,
share_rezero
:
bool
=
False
,
append_dense_inputs
:
bool
=
False
,
**
kwargs
):
super
().
__init__
(
**
kwargs
)
...
...
@@ -420,6 +421,7 @@ class FunnelTransformerEncoder(tf.keras.layers.Layer):
self
.
_pool_strides
=
pool_strides
# This is a list here.
self
.
_unpool_length
=
unpool_length
self
.
_pool_type
=
pool_type
self
.
_append_dense_inputs
=
append_dense_inputs
self
.
_config
=
{
'vocab_size'
:
vocab_size
,
...
...
@@ -485,11 +487,22 @@ class FunnelTransformerEncoder(tf.keras.layers.Layer):
word_embeddings
=
self
.
_embedding_layer
(
word_ids
)
if
dense_inputs
is
not
None
:
# Concat the dense embeddings at sequence begin so unpool_len can control
# embedding not being pooled.
word_embeddings
=
tf
.
concat
([
dense_inputs
,
word_embeddings
],
axis
=
1
)
type_ids
=
tf
.
concat
([
dense_type_ids
,
type_ids
],
axis
=
1
)
mask
=
tf
.
concat
([
dense_mask
,
mask
],
axis
=
1
)
# Allow concatenation of the dense embeddings at sequence end if requested
# and `unpool_length`` is set as zero
if
self
.
_append_dense_inputs
:
if
self
.
_unpool_length
!=
0
:
raise
ValueError
(
'unpool_length is not supported by append_dense_inputs now.'
)
word_embeddings
=
tf
.
concat
([
word_embeddings
,
dense_inputs
],
axis
=
1
)
type_ids
=
tf
.
concat
([
type_ids
,
dense_type_ids
],
axis
=
1
)
mask
=
tf
.
concat
([
mask
,
dense_mask
],
axis
=
1
)
else
:
# Concat the dense embeddings at sequence begin so unpool_len can
# control embedding not being pooled.
word_embeddings
=
tf
.
concat
([
dense_inputs
,
word_embeddings
],
axis
=
1
)
type_ids
=
tf
.
concat
([
dense_type_ids
,
type_ids
],
axis
=
1
)
mask
=
tf
.
concat
([
dense_mask
,
mask
],
axis
=
1
)
# absolute position embeddings
position_embeddings
=
self
.
_position_embedding_layer
(
word_embeddings
)
type_embeddings
=
self
.
_type_embedding_layer
(
type_ids
)
...
...
official/nlp/modeling/networks/funnel_transformer_test.py
浏览文件 @
ce8f9010
...
...
@@ -101,7 +101,11 @@ class FunnelTransformerEncoderTest(parameterized.TestCase, tf.test.TestCase):
self
.
assertAllEqual
(
tf
.
float32
,
data
.
dtype
)
self
.
assertAllEqual
(
pooled_dtype
,
pooled
.
dtype
)
def
test_network_creation_dense
(
self
):
@
parameterized
.
named_parameters
(
(
"append_dense_inputs"
,
True
),
(
"dense_inputs_at_sequence_begin"
,
False
),
)
def
test_network_creation_dense
(
self
,
append_dense_inputs
):
tf
.
keras
.
mixed_precision
.
set_global_policy
(
"mixed_float16"
)
pool_type
=
"avg"
...
...
@@ -120,7 +124,8 @@ class FunnelTransformerEncoderTest(parameterized.TestCase, tf.test.TestCase):
pool_type
=
pool_type
,
max_sequence_length
=
sequence_length
+
dense_sequence_length
,
unpool_length
=
0
,
transformer_cls
=
"TransformerEncoderBlock"
)
transformer_cls
=
"TransformerEncoderBlock"
,
append_dense_inputs
=
append_dense_inputs
)
# Create the inputs (note that the first dimension is implicit).
word_ids
=
tf
.
keras
.
Input
(
shape
=
(
sequence_length
,),
dtype
=
tf
.
int32
)
mask
=
tf
.
keras
.
Input
(
shape
=
(
sequence_length
,),
dtype
=
tf
.
int32
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录