/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.tika.langdetect.optimaize;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;

import org.apache.tika.langdetect.LanguageDetectorTest;
import org.apache.tika.language.detect.LanguageConfidence;
import org.apache.tika.language.detect.LanguageDetector;
import org.apache.tika.language.detect.LanguageResult;
import org.apache.tika.language.detect.LanguageWriter;

public class OptimaizeLangDetectorTest extends LanguageDetectorTest {
    /*
     * The complete list of supported languages (as of 0.5) is below.
     * The ones we have tests for have '*' after the name.
     *
    af Afrikaans
    an Aragonese
    ar Arabic
    ast Asturian
    be Belarusian
    br Breton
    ca Catalan
    bg Bulgarian
    bn Bengali
    cs Czech
    cy Welsh
    da Danish *
    de German *
    el Greek *
    en English *
    es Spanish *
    et Estonian
    eu Basque
    fa Persian
    fi Finnish *
    fr French *
    ga Irish
    gl Galician
    gu Gujarati
    he Hebrew
    hi Hindi
    hr Croatian
    ht Haitian
    hu Hungarian
    id Indonesian
    is Icelandic
    it Italian *
    ja Japanese *
    km Khmer
    kn Kannada
    ko Korean
    lt Lithuanian *
    lv Latvian
    mk Macedonian
    ml Malayalam
    mr Marathi
    ms Malay
    mt Maltese
    ne Nepali
    nl Dutch *
    no Norwegian
    oc Occitan
    pa Punjabi
    pl Polish
    pt Portuguese *
    ro Romanian
    ru Russian
    sk Slovak
    sl Slovene
    so Somali
    sq Albanian
    sr Serbian
    sv Swedish *
    sw Swahili
    ta Tamil
    te Telugu
    th Thai *
    tl Tagalog
    tr Turkish
    uk Ukrainian
    ur Urdu
    vi Vietnamese
    yi Yiddish
    zh-CN Simplified Chinese * (just generic Chinese)
    zh-TW Traditional Chinese * (just generic Chinese)
    */

    /**
     * Test correct detection for the many (short) translations of the
     * "Universal Declaration of Human Rights (Article 1)", at
     * http://www.omniglot.com/udhr
     * <p>
     * Also make sure we get uncertain results for some set of unsupported
     * languages.
     *
     * @throws Exception
     */
    @Test
    public void testUniversalDeclarationOfHumanRights() throws Exception {
        LanguageDetector detector = new OptimaizeLangDetector();
        detector.loadModels();

        LanguageWriter writer = new LanguageWriter(detector);

        Map<String, String> knownText = getTestLanguages("udhr-known.txt");
        for (String language : knownText.keySet()) {
            writer.reset();
            writer.append(knownText.get(language));

            LanguageResult result = detector.detect();
            assertNotNull(result);

            assertEquals(language, result.getLanguage());
            // System.out.println(String.format("'%s': %s (%f)", language,
            // result.getConfidence(), result.getRawScore()));
        }

        Map<String, String> unknownText = getTestLanguages("udhr-unknown.txt");
        for (String language : unknownText.keySet()) {
            writer.reset();
            writer.append(unknownText.get(language));

            LanguageResult result = detector.detect();
            if (result != null) {
                assertFalse(result.isReasonablyCertain());
                // System.out.println(String.format("Looking for '%s', got '%s': %s (%f)",
                // language, result.getLanguage(), result.getConfidence(), result.getRawScore()));
            }
        }

        writer.close();
    }

    @Test
    public void testAllLanguages() throws IOException {
        LanguageDetector detector = new OptimaizeLangDetector();
        detector.loadModels();

        LanguageWriter writer = new LanguageWriter(detector);

        for (String language : getTestLanguages()) {
            writer.reset();

            writeTo(language, writer);

            LanguageResult result = detector.detect();
            assertNotNull(result);

            assertTrue(result.isLanguage(language));
            assertTrue(result.isReasonablyCertain());
        }
    }

    @Test
    public void testMixedLanguages() throws IOException {
        LanguageDetector detector = new OptimaizeLangDetector().setMixedLanguages(true);

        detector.loadModels();
        LanguageWriter writer = new LanguageWriter(detector);

        String[] languages = getTestLanguages();
        for (int i = 0; i < languages.length; i++) {
            String language = languages[i];
            for (int j = i + 1; j < languages.length; j++) {
                String other = languages[j];

                writer.reset();
                writeTo(language, writer);
                writeTo(other, writer);

                List<LanguageResult> results = detector.detectAll();
                if (results.size() > 0) {
                    LanguageResult result = results.get(0);

                    assertFalse(result.isReasonablyCertain(),
                            "mix of " + language + " and " + other + " incorrectly detected as " +
                                    result);
                }
            }
        }

        writer.close();
    }

    @Test
    public void testShortText() throws IOException {
        LanguageDetector detector = new OptimaizeLangDetector().setShortText(true).loadModels();

        // First verify that we get no result with empty or very short text.
        LanguageWriter writer = new LanguageWriter(detector);
        writer.append("");
        assertEquals(LanguageConfidence.NONE, detector.detect().getConfidence());

        writer.reset();
        writer.append("  ");
        assertEquals(LanguageConfidence.NONE, detector.detect().getConfidence());

        for (String language : getTestLanguages()) {
            // Short pieces of Japanese are detected as Chinese
            if (language.equals("ja")) {
                continue;
            }

            // We need at least 300 characters to detect Chinese reliably.
            writer.reset();
            writeTo(language, writer, 300);

            LanguageResult result = detector.detect();
            assertNotNull(result, String.format(Locale.US, "Language '%s' wasn't detected",
                            language));

            assertTrue(result.isLanguage(language), String.format(Locale.US, "Language '%s' was " +
                            "detected as '%s'", language,
                    result.getLanguage()));
            assertTrue(result.isReasonablyCertain(),
                    String.format(Locale.US, "Language '%s' isn't reasonably certain: %s", language,
                            result.getConfidence()));
        }

        writer.close();
    }

    private Map<String, String> getTestLanguages(String resourceName) throws IOException {
        Map<String, String> result = new HashMap<>();
        List<String> languages =
                IOUtils.readLines(OptimaizeLangDetectorTest.class.getResourceAsStream(resourceName),
                        StandardCharsets.UTF_8);
        for (String line : languages) {
            line = line.trim();
            if (line.isEmpty() || line.startsWith("#")) {
                continue;
            }

            String[] pieces = line.split("\t", 2);
            if (pieces.length != 2) {
                throw new IllegalArgumentException("Invalid language data line: " + line);
            }

            result.put(pieces[0], pieces[1]);
        }

        return result;
    }

    @Test
    @Timeout(5000)
    public void testOptimaizeRegexBug() throws Exception {
        //confirm TIKA-2777 doesn't affect langdetect's Optimaize
        LanguageDetector detector = new OptimaizeLangDetector().setShortText(false).loadModels();
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < 50000; i++) {
            sb.append('a');
        }
        detector.detect(sb.toString());
    }

}
