# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import re
from typing import List

import codecs
import jieba

jieba.setLogLevel(logging.INFO)

from sacremoses import MosesTokenizer, MosesDetokenizer
from subword_nmt.apply_bpe import BPE


class MTTokenizer(object):
    def __init__(self, bpe_codes_file: str, lang_src: str = 'zh', lang_trg: str = 'en', separator='@@'):
        self.moses_detokenizer = MosesDetokenizer(lang=lang_trg)
        self.bpe_tokenizer = BPE(codes=codecs.open(bpe_codes_file, encoding='utf-8'),
                                 merges=-1,
                                 separator=separator,
                                 vocab=None,
                                 glossaries=None)

    def tokenize(self, text: str):
        """
        Convert source string into bpe tokens.
        """
        text = text.replace(' ', '')  # Remove blanks in Chinese text.
        jieba_tokens = list(jieba.cut(text))
        tokenized_text = ' '.join(jieba_tokens)
        tokenized_bpe_text = self.bpe_tokenizer.process_line(tokenized_text)  # Apply bpe to text
        bpe_tokens = tokenized_bpe_text.split(' ')
        return bpe_tokens

    def detokenize(self, tokens: List[str]):
        """
        Convert target bpe tokens into string.
        """
        separator = self.bpe_tokenizer.separator
        text_with_separators = ' '.join(tokens)
        clean_text = re.sub(f'({separator} )|({separator} ?$)', '', text_with_separators)
        clean_tokens = clean_text.split(' ')
        detokenized_text = self.moses_detokenizer.tokenize(clean_tokens, return_str=True)
        return detokenized_text


def post_process_seq(seq, bos_idx, eos_idx, output_bos=False, output_eos=False):
    """
    Post-process the decoded sequence.
    """
    eos_pos = len(seq) - 1
    for i, idx in enumerate(seq):
        if idx == eos_idx:
            eos_pos = i
            break
    seq = [int(idx) for idx in seq[:eos_pos + 1] if (output_bos or idx != bos_idx) and (output_eos or idx != eos_idx)]
    return seq