/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.index;

import java.io.IOException;
import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
import org.apache.lucene.tests.store.MockDirectoryWrapper;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;

public class TestIndexWriterForceMerge extends LuceneTestCase {
  public void testPartialMerge() throws IOException {

    Directory dir = newDirectory();

    final Document doc = new Document();
    doc.add(newStringField("content", "aaa", Field.Store.NO));
    final int incrMin = TEST_NIGHTLY ? 15 : 40;
    for (int numDocs = 10;
        numDocs < 500;
        numDocs += TestUtil.nextInt(random(), incrMin, 5 * incrMin)) {
      LogDocMergePolicy ldmp = new LogDocMergePolicy();
      ldmp.setMinMergeDocs(1);
      ldmp.setMergeFactor(5);
      IndexWriter writer =
          new IndexWriter(
              dir,
              newIndexWriterConfig(new MockAnalyzer(random()))
                  .setOpenMode(OpenMode.CREATE)
                  .setMaxBufferedDocs(2)
                  .setMergePolicy(ldmp));
      for (int j = 0; j < numDocs; j++) writer.addDocument(doc);
      writer.close();

      SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
      final int segCount = sis.size();

      ldmp = new LogDocMergePolicy();
      ldmp.setMergeFactor(5);
      writer =
          new IndexWriter(
              dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(ldmp));
      writer.forceMerge(3);
      writer.close();

      sis = SegmentInfos.readLatestCommit(dir);
      final int optSegCount = sis.size();

      if (segCount < 3) assertEquals(segCount, optSegCount);
      else assertEquals(3, optSegCount);
    }
    dir.close();
  }

  public void testMaxNumSegments2() throws IOException {
    Directory dir = newDirectory();

    final Document doc = new Document();
    doc.add(newStringField("content", "aaa", Field.Store.NO));

    LogDocMergePolicy ldmp = new LogDocMergePolicy();
    ldmp.setMinMergeDocs(1);
    ldmp.setMergeFactor(4);
    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setMergePolicy(ldmp)
                .setMergeScheduler(new ConcurrentMergeScheduler()));

    for (int iter = 0; iter < 10; iter++) {
      for (int i = 0; i < 19; i++) writer.addDocument(doc);

      writer.commit();
      writer.waitForMerges();
      writer.commit();

      SegmentInfos sis = SegmentInfos.readLatestCommit(dir);

      final int segCount = sis.size();
      writer.forceMerge(7);
      writer.commit();
      writer.waitForMerges();

      sis = SegmentInfos.readLatestCommit(dir);
      final int optSegCount = sis.size();

      if (segCount < 7) assertEquals(segCount, optSegCount);
      else assertEquals("seg: " + segCount, 7, optSegCount);
    }
    writer.close();
    dir.close();
  }

  /**
   * Make sure forceMerge doesn't use any more than 1X starting index size as its temporary free
   * space required.
   */
  public void testForceMergeTempSpaceUsage() throws IOException {

    final MockDirectoryWrapper dir = newMockDirectory();
    // don't use MockAnalyzer, variable length payloads can cause merge to make things bigger,
    // since things are optimized for fixed length case. this is a problem for MemoryPF's encoding.
    // (it might have other problems too)
    Analyzer analyzer =
        new Analyzer() {
          @Override
          protected TokenStreamComponents createComponents(String fieldName) {
            return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
          }
        };
    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(analyzer)
                .setMaxBufferedDocs(10)
                .setMergePolicy(newLogMergePolicy()));

    if (VERBOSE) {
      System.out.println("TEST: config1=" + writer.getConfig());
    }

    for (int j = 0; j < 500; j++) {
      TestIndexWriter.addDocWithIndex(writer, j);
    }
    // force one extra segment w/ different doc store so
    // we see the doc stores get merged
    writer.commit();
    TestIndexWriter.addDocWithIndex(writer, 500);
    writer.close();

    long startDiskUsage = 0;
    for (String f : dir.listAll()) {
      startDiskUsage += dir.fileLength(f);
      if (VERBOSE) {
        System.out.println(f + ": " + dir.fileLength(f));
      }
    }
    if (VERBOSE) {
      System.out.println("TEST: start disk usage = " + startDiskUsage);
    }
    String startListing = listFiles(dir);

    dir.resetMaxUsedSizeInBytes();
    dir.setTrackDiskUsage(true);

    writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.APPEND)
                .setMergePolicy(newLogMergePolicy()));

    if (VERBOSE) {
      System.out.println("TEST: config2=" + writer.getConfig());
    }

    writer.forceMerge(1);
    writer.close();

    long finalDiskUsage = 0;
    for (String f : dir.listAll()) {
      finalDiskUsage += dir.fileLength(f);
      if (VERBOSE) {
        System.out.println(f + ": " + dir.fileLength(f));
      }
    }
    if (VERBOSE) {
      System.out.println("TEST: final disk usage = " + finalDiskUsage);
    }

    // The result of the merged index is often smaller, but sometimes it could
    // be bigger (compression slightly changes, Codec changes etc.). Therefore
    // we compare the temp space used to the max of the initial and final index
    // size
    long maxStartFinalDiskUsage = Math.max(startDiskUsage, finalDiskUsage);
    long maxDiskUsage = dir.getMaxUsedSizeInBytes();
    assertTrue(
        "forceMerge used too much temporary space: starting usage was "
            + startDiskUsage
            + " bytes; final usage was "
            + finalDiskUsage
            + " bytes; max temp usage was "
            + maxDiskUsage
            + " but should have been at most "
            + (4 * maxStartFinalDiskUsage)
            + " (= 4X starting usage), BEFORE="
            + startListing
            + "AFTER="
            + listFiles(dir),
        maxDiskUsage <= 4 * maxStartFinalDiskUsage);
    dir.close();
  }

  // print out listing of files and sizes, but recurse into CFS to debug nested files there.
  private String listFiles(Directory dir) throws IOException {
    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
    StringBuilder sb = new StringBuilder();
    sb.append(System.lineSeparator());
    for (SegmentCommitInfo info : infos) {
      for (String file : info.files()) {
        sb.append(String.format(Locale.ROOT, "%-20s%d%n", file, dir.fileLength(file)));
      }
      if (info.info.getUseCompoundFile()) {
        try (Directory cfs =
            info.info
                .getCodec()
                .compoundFormat()
                .getCompoundReader(dir, info.info, IOContext.DEFAULT)) {
          for (String file : cfs.listAll()) {
            sb.append(
                String.format(
                    Locale.ROOT,
                    " |- (inside compound file) %-20s%d%n",
                    file,
                    cfs.fileLength(file)));
          }
        }
      }
    }
    sb.append(System.lineSeparator());
    return sb.toString();
  }

  // Test calling forceMerge(1, false) whereby forceMerge is kicked
  // off but we don't wait for it to finish (but
  // writer.close()) does wait
  public void testBackgroundForceMerge() throws IOException {

    Directory dir = newDirectory();
    for (int pass = 0; pass < 2; pass++) {
      IndexWriter writer =
          new IndexWriter(
              dir,
              newIndexWriterConfig(new MockAnalyzer(random()))
                  .setOpenMode(OpenMode.CREATE)
                  .setMaxBufferedDocs(2)
                  .setMergePolicy(newLogMergePolicy(51)));
      Document doc = new Document();
      doc.add(newStringField("field", "aaa", Field.Store.NO));
      for (int i = 0; i < 100; i++) {
        writer.addDocument(doc);
      }
      writer.forceMerge(1, false);

      if (0 == pass) {
        writer.close();
        DirectoryReader reader = DirectoryReader.open(dir);
        assertEquals(1, reader.leaves().size());
        reader.close();
      } else {
        // Get another segment to flush so we can verify it is
        // NOT included in the merging
        writer.addDocument(doc);
        writer.addDocument(doc);
        writer.close();

        DirectoryReader reader = DirectoryReader.open(dir);
        assertTrue(reader.leaves().size() > 1);
        reader.close();

        SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
        assertEquals(2, infos.size());
      }
    }

    dir.close();
  }
}
