# encoding: utf-8

#  Copyright 2015-present The Scikit Flow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf

from tensorflow.contrib.learn.python.learn.preprocessing import text
from tensorflow.contrib.learn.python.learn.preprocessing import CategoricalVocabulary


class TextTest(tf.test.TestCase):

    def testTokenizer(self):
        words = text.tokenizer(["a b c", "a\nb\nc", "a, b - c",
                                "фыв выф", "你好 怎么样"])
        self.assertEqual(list(words),
                         [["a", "b", "c"],
                          ["a", "b", "c"],
                          ["a", "b", "-", "c"],
                          ["фыв", "выф"],
                          ["你好", "怎么样"]])

    def testByteProcessor(self):
        processor = text.ByteProcessor(max_document_length=8)
        inp = ["abc", "фыва", "фыва", b"abc",
               "12345678901234567890"]
        res = list(processor.fit_transform(inp))
        self.assertAllEqual(res,
                            [[97, 98, 99, 0, 0, 0, 0, 0],
                            [209, 132, 209, 139, 208, 178, 208, 176],
                            [209, 132, 209, 139, 208, 178, 208, 176],
                            [97, 98, 99, 0, 0, 0, 0, 0],
                            [49, 50, 51, 52, 53, 54, 55, 56]])
        res = list(processor.reverse(res))
        self.assertAllEqual(res,
            ["abc", "фыва", "фыва", "abc", "12345678"])

    def testVocabularyProcessor(self):
        vocab_processor = text.VocabularyProcessor(
            max_document_length=4,
            min_frequency=1)
        tokens = vocab_processor.fit_transform(
            ["a b c", "a\nb\nc", "a, b - c"])
        self.assertAllEqual(list(tokens),
                            [[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 3]])

    def testVocabularyProcessorSaveRestore(self):
        filename = tf.test.get_temp_dir() + 'test.vocab'
        vocab_processor = text.VocabularyProcessor(
            max_document_length=4,
            min_frequency=1)
        tokens = vocab_processor.fit_transform(
            ["a b c", "a\nb\nc", "a, b - c"])
        vocab_processor.save(filename)
        new_vocab = text.VocabularyProcessor.restore(filename)
        tokens = new_vocab.transform(["a b c"])
        self.assertAllEqual(list(tokens), [[1, 2, 3, 0]])

    def testExistingVocabularyProcessor(self):
        vocab = CategoricalVocabulary()
        vocab.get("A")
        vocab.get("B")
        vocab.freeze()
        vocab_processor = text.VocabularyProcessor(
            max_document_length=4, vocabulary=vocab, tokenizer_fn=list)
        tokens = vocab_processor.fit_transform(["ABC", "CBABAF"])
        self.assertAllEqual(list(tokens), [[1, 2, 0, 0], [0, 2, 1, 2]])


if __name__ == "__main__":
    tf.test.main()
