From 45fd32c82e1bb55f8eac44d77d40902302c5aee2 Mon Sep 17 00:00:00 2001 From: Hongkun Yu Date: Wed, 28 Sep 2022 15:57:45 -0700 Subject: [PATCH] Internal change PiperOrigin-RevId: 477574174 --- official/nlp/modeling/layers/transformer_scaffold.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/official/nlp/modeling/layers/transformer_scaffold.py b/official/nlp/modeling/layers/transformer_scaffold.py index 8ce8bcf80..6b46a4b81 100644 --- a/official/nlp/modeling/layers/transformer_scaffold.py +++ b/official/nlp/modeling/layers/transformer_scaffold.py @@ -335,7 +335,9 @@ class TransformerScaffold(tf.keras.layers.Layer): training=training) layer_output += source_attention_output else: - # if not norm_first, assume that the feedforwad does apply layer norm + # Attention: if not norm_first, assume that the feedforwad does apply + # layer norm. The feedford also apply residual connection. Please + # read the `GatedFeedforward` as a concrete example. layer_output = self._feedforward_block(attention_output, training=training) -- GitLab