Change the split level of poem sentences

generate_chinese_poetry/generate.py

@@ -82,7 +82,7 @@ def generate(model_path, word_dict_path, test_data_path, batch_size, beam_size,
         decoder_hidden_dim=512,
         bos_id=0,
         eos_id=1,
-        max_length=17,
+        max_length=9,
         beam_size=beam_size,
         is_generating=True)
 

generate_chinese_poetry/preprocess.py

@@ -52,7 +52,7 @@ def preprocess(datadir, outfile, dictfile):
                         u"！" in p or u"？" in p or u"●" in p or u"□" in p or
                         u"囗" in p or u"）" in p):
                     continue
-                paragraphs = p.split(u"。")
+                paragraphs = re.split(u"。|，", p)
                 paragraphs = filter(lambda x: len(x), paragraphs)
                 if len(paragraphs) > 1:
                     dataset.append((title, author, paragraphs))

generate_chinese_poetry/train.py

@@ -90,7 +90,7 @@ def train(num_passes,
         decoder_hidden_dim=512,
         bos_id=0,
         eos_id=1,
-        max_length=17)
+        max_length=9)
 
     parameters = paddle.parameters.create(cost)
     if init_model_path: