/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.netbeans.modules.subversion;

import org.netbeans.modules.versioning.util.FileUtils;
import org.netbeans.modules.turbo.CacheIndex;
import org.netbeans.modules.subversion.util.*;
import org.netbeans.modules.turbo.TurboProvider;
import java.io.*;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.openide.modules.Places;
import org.openide.util.NbBundle;

/**
 * Storage of file attributes with shortcut to retrieve all stored values.
 *
 * @author Maros Sandor
 */
class DiskMapTurboProvider implements TurboProvider {

    static final String ATTR_STATUS_MAP = "subversion.STATUS_MAP";  // NOI18N

    private static final int STATUS_VALUABLE = FileInformation.STATUS_MANAGED
            & ~FileInformation.STATUS_VERSIONED_UPTODATE & ~FileInformation.STATUS_NOTVERSIONED_EXCLUDED;
    private static final String CACHE_DIRECTORY = "svncache"; // NOI18N
    private static final int DIRECTORY = Integer.highestOneBit(Integer.MAX_VALUE);
    private static final Logger LOG = Logger.getLogger(DiskMapTurboProvider.class.getName());

    private File                            cacheStore;

    private final CacheIndex index = createCacheIndex();
    private final CacheIndex conflictedIndex = createCacheIndex();
    private final CacheIndex ignoresIndex = createCacheIndex();
    private final boolean logModifiedFiles = Boolean.getBoolean("versioning.subversion.turbo.logModifiedFiles"); //NOI18N

    DiskMapTurboProvider() {
        initCacheStore();
    }

    File[] getIndexValues(File file, int includeStatus) {
        if (includeStatus == FileInformation.STATUS_VERSIONED_CONFLICT) {
            return conflictedIndex.get(file);
        } else if (includeStatus == FileInformation.STATUS_NOTVERSIONED_EXCLUDED) {
            return ignoresIndex.get(file);
        } else if ((includeStatus & FileInformation.STATUS_NOTVERSIONED_EXCLUDED) != 0) {
            File[] files = index.get(file);
            File[] ignores = ignoresIndex.get(file);
            return mergeArrays(files, ignores);
        } else {
            return index.get(file);
        }
    }

    File[] getAllIndexValues() {
        File[] files = index.getAllValues();
        File[] ignores = ignoresIndex.getAllValues();
        return mergeArrays(files, ignores);
    }

    private File[] mergeArrays (File[] arr1, File[] arr2) {
        if (arr1.length == 0) {
            return arr2;
        } else if (arr2.length == 0) {
            return arr1;
        } else {
            Set<File> merged = new HashSet<>(Arrays.asList(arr2));
            merged.addAll(Arrays.asList(arr1));
            return merged.toArray(new File[merged.size()]);
        }
    }

    public void computeIndex() {
        long ts = System.currentTimeMillis();
        long entriesCount = 0;
        long failedReadCount = 0;
        try {
            if (!cacheStore.isDirectory()) {
                cacheStore.mkdirs();
            }
            
            File [] files;
            synchronized(this) {
                files = cacheStore.listFiles();
            }

            if(files == null) {
                return;
            }
            
            int modifiedFiles = 0;
            int locallyNewFiles = 0;
            Map<String, Integer> locallyNewFolders = new HashMap<String, Integer>();
            Map<String, Integer> modifiedFolders = new HashMap<String, Integer>();

            for (int i = 0; i < files.length; i++) {
                File file = files[i];
                synchronized(this) {
                    if (file.getName().endsWith(".bin") == false) { // NOI18N
                        // on windows list returns already deleted .new files
                        continue;
                    }
                    boolean readFailed = false;
                    int itemIndex = -1;
                    DataInputStream dis = null;
                    try {
                        int retry = 0;
                        while (true) {
                            try {
                                dis = new DataInputStream(new BufferedInputStream(new FileInputStream(file)));
                                break;
                            } catch (IOException ioex) {
                                retry++;
                                if (retry > 7) {
                                    throw ioex;
                                }
                                Thread.sleep(retry * 30);
                            }
                        }

                        itemIndex = 0;
                        for (;;) {
                            ++itemIndex;
                            int pathLen;
                            try {
                                pathLen = dis.readInt();
                            } catch (EOFException e) {
                                // reached EOF, no entry for this key
                                break;
                            }
                            dis.readInt();
                            String path = readChars(dis, pathLen);
                            Map value = readValue(dis, path);
                            for (Iterator j = value.keySet().iterator(); j.hasNext();) {
                                entriesCount++;
                                File f = (File) j.next();
                                FileInformation info = (FileInformation) value.get(f);
                                if((info.getStatus() & FileInformation.STATUS_VERSIONED_CONFLICT) != 0) {
                                    conflictedIndex.add(f);
                                }
                                if ((info.getStatus() & STATUS_VALUABLE) != 0) {
                                    index.add(f);
                                    modifiedFiles++;
                                    addModifiedFile(modifiedFolders, f);
                                    if ((info.getStatus() & FileInformation.STATUS_NOTVERSIONED_NEWLOCALLY) != 0) {
                                        locallyNewFiles++;
                                        addLocallyNewFile(locallyNewFolders, info.isDirectory() ? f.getAbsolutePath() : f.getParent());
                                    }
                                }
                            }
                        }
                    } catch (EOFException e) {
                        logCorruptedCacheFile(file, itemIndex, e);
                        readFailed = true;
                    } catch (Exception e) {
                        Subversion.LOG.log(Level.SEVERE, null, e);
                    } finally {
                        if (dis != null) try { dis.close(); } catch (IOException e) {}
                    }
                    if (readFailed) {
                        // cache file is corrupted, delete it (will be recreated on-demand later)
                        file.delete();
                        failedReadCount++;
                    }
                }
            }
            if (locallyNewFiles > 1000) {
                logTooManyNewFiles(locallyNewFolders, locallyNewFiles);
            } else if (modifiedFiles > 5000) {
                logTooManyModifications(modifiedFolders, modifiedFiles);
            }
        } finally {
            Subversion.LOG.log(Level.INFO, "Finished indexing svn cache with {0} entries. Elapsed time: {1} ms.", new Object[]{entriesCount, System.currentTimeMillis() - ts});
            if(failedReadCount > 0) {
                Subversion.LOG.log(Level.INFO, " read failed {0} times.", failedReadCount);
            }
        }
    }

    @Override
    public boolean recognizesAttribute(String name) {
        return ATTR_STATUS_MAP.equals(name);
    }

    @Override
    public boolean recognizesEntity(Object key) {
        return key instanceof File;
    }

    @Override
    public synchronized Object readEntry(Object key, String name, MemoryCache memoryCache) {
        assert key instanceof File;
        assert name != null;

        boolean readFailed = false;
        File dir = (File) key;
        File store = getStore(dir);
        if (!store.isFile()) {
            return null;
        }

        String dirPath = dir.getAbsolutePath();
        int dirPathLen = dirPath.length();
        DataInputStream dis = null;
        int itemIndex = -1;
        try {

            int retry = 0;
            while (true) {
                try {
                    dis = new DataInputStream(new BufferedInputStream(new FileInputStream(store)));
                    break;
                } catch (IOException ioex) {
                    retry++;
                    if (retry > 7) {
                        throw ioex;
                    }
                    Thread.sleep(retry * 30);
                }
            }

            itemIndex = 0;
            for (;;) {
                ++itemIndex;
                int pathLen;
                try {
                    pathLen = dis.readInt();
                } catch (EOFException e) {
                    // reached EOF, no entry for this key
                    break;
                }
                int mapLen = dis.readInt();
                if (pathLen != dirPathLen) {
                    skip(dis, pathLen * 2 + mapLen);
                } else {
                    String path = readChars(dis, pathLen);
                    if (dirPath.equals(path)) {
                        return readValue(dis, path);
                    } else {
                        skip(dis, mapLen);
                    }
                }
            }
        } catch (EOFException e) {
            logCorruptedCacheFile(store, itemIndex, e);
            readFailed = true;
        } catch (Exception e) {
            Subversion.LOG.log(Level.INFO, e.getMessage(), e);
            readFailed = true;
        } finally {
            if (dis != null) try { dis.close(); } catch (IOException e) {}
        }
        if (readFailed) store.delete(); // cache file is corrupted, delete it (will be recreated on-demand later)
        return null;
    }

    @Override
    public synchronized boolean writeEntry(Object key, String name, Object value) {
        assert key instanceof File;
        assert name != null;

        if (value != null) {
            if (!(value instanceof Map)) return false;
            if (!isValuable(value)) value = null;
        }

        File dir = (File) key;
        String dirPath = dir.getAbsolutePath();
        int dirPathLen = dirPath.length();
        File store = getStore(dir);

        if (value == null && !store.exists()) return true;

        File storeNew = new File(store.getParentFile(), store.getName() + ".new"); // NOI18N
        if (!cacheStore.isDirectory()) {
            cacheStore.mkdirs();
        }

        DataOutputStream oos = null;
        DataInputStream dis = null;
        boolean readFailed = false;
        int itemIndex = -1;
        try {
            oos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(storeNew)));
            if (value != null) {
                writeEntry(oos, dirPath, value);
            }
            if (store.exists()) {
                int retry = 0;
                while (true) {
                    try {
                        dis = new DataInputStream(new BufferedInputStream(new FileInputStream(store)));
                        break;
                    } catch (IOException ioex) {
                        retry++;
                        if (retry > 7) {
                            throw ioex;
                        }
                        Thread.sleep(retry * 30);
                    }
                }

                itemIndex = 0;
                for (;;) {
                    ++itemIndex;
                    int pathLen;
                    try {
                        pathLen = dis.readInt();
                    } catch (EOFException e) {
                        break;
                    }
                    int mapLen = dis.readInt();
                    if (pathLen == dirPathLen) {
                        String path = readChars(dis, pathLen);
                        if (dirPath.equals(path)) {
                            skip(dis, mapLen);
                        } else {
                            oos.writeInt(pathLen);
                            oos.writeInt(mapLen);
                            oos.writeChars(path);
                            copyStreams(oos, dis, mapLen);
                        }
                    } else {
                        oos.writeInt(pathLen);
                        oos.writeInt(mapLen);
                        copyStreams(oos, dis, mapLen + pathLen * 2);
                    }
                }
            }
        } catch (EOFException e) {
            logCorruptedCacheFile(store, itemIndex, e);
            readFailed = true;
        } catch (FileNotFoundException ex) {
            Subversion.LOG.log(Level.INFO, "File could not be created, check if you are running only a single instance of netbeans for this userdir", ex); //NOI18N
            return true;
        } catch (Exception e) {
            Subversion.LOG.log(Level.INFO, "Copy: " + store.getAbsolutePath() + " to: " + storeNew.getAbsolutePath(), e);  // NOI18N
            return true;
        } finally {
            if (oos != null) try { oos.close(); } catch (IOException e) {}
            if (dis != null) try { dis.close(); } catch (IOException e) {}
        }
        adjustIndex(dir, value);

        if (readFailed) {
            store.delete(); // cache file is corrupted, delete it (will be recreated on-demand later)
            return true;
        }
        try {
            FileUtils.renameFile(storeNew, store);
        } catch (FileNotFoundException ex) {
            Subversion.LOG.log(Level.INFO, 
                    "File could not be renamed, check if you are running only a single instance of netbeans for this userdir", //NOI18N
                    ex);
        } catch (IOException ex) {
            Subversion.LOG.log(Level.SEVERE, null, ex);
        }
        return true;
    }

    private void adjustIndex(File dir, Object value) {
        // the file must be a folder or must not exist
        // adding existing file is forbidden
        assert !dir.isFile();
        Map map = (Map) value;
        Set set = map != null ? map.keySet() : null;

        // all modified files
        Set<File> conflictedSet = new HashSet<File>();
        Set<File> newSet = new HashSet<File>();
        Set<File> ignoredSet = new HashSet<File>();
        if(set != null) {
            for (Iterator i = set.iterator(); i.hasNext();) {
                File file = (File) i.next();
                FileInformation info = (FileInformation) map.get(file);

                // conflict
                if((info.getStatus() & FileInformation.STATUS_VERSIONED_CONFLICT) != 0) {
                    conflictedSet.add(file);
                }

                if (info.getStatus() == FileInformation.STATUS_NOTVERSIONED_EXCLUDED) {
                    ignoredSet.add(file);
                } else {
                    if ((info.getStatus() & FileInformation.STATUS_NOTVERSIONED_EXCLUDED) != 0) {
                        // this can hardly happen
                        assert false;
                        ignoredSet.add(file);
                    }
                    // all but uptodate
                    if((info.getStatus() & STATUS_VALUABLE) != 0) {
                        newSet.add(file);
                    }
                }
            }
        }
        index.add(dir, newSet);
        ignoresIndex.add(dir, ignoredSet);
        conflictedIndex.add(dir, conflictedSet);
    }

    /**
     * Logs the EOFException and the corrupted cache file
     * @param file file which caused the error
     * @param itemIndex a position in the file when the error showed
     * @param e
     */
    private void logCorruptedCacheFile(File file, int itemIndex, EOFException e) {
        try {
            File tmpFile = File.createTempFile("svn_", ".bin");
            Subversion.LOG.log(Level.INFO, "Corrupted cache file " + file.getAbsolutePath() + " at position " + itemIndex, e);
            FileUtils.copyFile(file, tmpFile);
            byte[] contents = FileUtils.getFileContentsAsByteArray(tmpFile);
            Subversion.LOG.log(Level.INFO, "Corrupted cache file length: {0}", contents.length);
            String encodedContent = Base64.getEncoder().encodeToString(contents); // log the file contents
            Subversion.LOG.log(Level.INFO, "Corrupted cache file content:\n{0}\n", encodedContent);
            Exception ex = new Exception("Corrupted cache file \"" + file.getAbsolutePath() + "\", please report in subversion module issues and attach "
                    + tmpFile.getAbsolutePath() + " plus the IDE message log", e);
            Subversion.LOG.log(Level.INFO, null, ex);
        } catch (IOException ex) {
            Subversion.LOG.log(Level.SEVERE, null, ex);
        }
    }

    private void skip(InputStream is, long len) throws IOException {
        while (len > 0) {
            long n = is.skip(len);
            if (n < 0) throw new EOFException("Missing " + len + " bytes.");  // NOI18N
            len -= n;
        }
    }

    private String readChars(DataInputStream dis, int len) throws IOException {
        if (len < 0 || len > 1024 * 1024 * 10) throw new EOFException("Len: " + len); // preventing from OOME
        StringBuilder sb = new StringBuilder(len);
        while (len-- > 0) {
            sb.append(dis.readChar());
        }
        return sb.toString();
    }

    private Map<File, FileInformation> readValue(DataInputStream dis, String dirPath) throws IOException {
        Map<File, FileInformation> map = new HashMap<File, FileInformation>();
        int len = dis.readInt();
        while (len-- > 0) {
            int nameLen = dis.readInt();
            String name = readChars(dis, nameLen);
            File file = new File(dirPath, name);
            int status = dis.readInt();
            FileInformation info = new FileInformation(status & (DIRECTORY - 1), status > (DIRECTORY - 1));
            map.put(file, info);
        }
        return map;
    }

    private void writeEntry(DataOutputStream dos, String dirPath, Object value) throws IOException {

        Map map = (Map) value;
        Set set = map.keySet();
        ByteArrayOutputStream baos = new ByteArrayOutputStream(set.size() * 50);
        DataOutputStream temp = new DataOutputStream(baos);

        temp.writeInt(set.size());
        for (Iterator i = set.iterator(); i.hasNext();) {
            File file = (File) i.next();
            FileInformation info = (FileInformation) map.get(file);
            temp.writeInt(file.getName().length());
            temp.writeChars(file.getName());
            temp.writeInt(info.getStatus() + (info.isDirectory() ? DIRECTORY : 0));
        }
        temp.close();
        byte [] valueBytes = baos.toByteArray();

        dos.writeInt(dirPath.length());
        dos.writeInt(valueBytes.length);
        dos.writeChars(dirPath);
        dos.write(valueBytes);
    }

    private boolean isValuable(Object value) {
        Map map = (Map) value;
        for (Iterator i = map.values().iterator(); i.hasNext();) {
            FileInformation info = (FileInformation) i.next();
            if ((info.getStatus() & STATUS_VALUABLE) != 0) return true;
        }
        return false;
    }

    private File getStore(File dir) {
        String dirPath = dir.getAbsolutePath();
        int dirHash = dirPath.hashCode();
        return new File(cacheStore, Integer.toString(dirHash % 173 + 172) + ".bin"); // NOI18N
    }

    private void initCacheStore() {
        cacheStore = Places.getCacheSubdirectory(CACHE_DIRECTORY);
    }

    private static void copyStreams(OutputStream out, InputStream in, int len) throws IOException {
        byte [] buffer = new byte[4096];
        int totalLen = len;
        for (;;) {
            int n = (len >= 0 && len <= 4096) ? len : 4096;
            n = in.read(buffer, 0, n);
            if (n < 0) throw new EOFException("Missing " + len + " bytes from total " + totalLen + " bytes.");  // NOI18N
            out.write(buffer, 0, n);
            if ((len -= n) == 0) break;
        }
        out.flush();
    }

    private static CacheIndex createCacheIndex() {
        return new CacheIndex() {
            @Override
            protected boolean isManaged(File file) {
                return SvnUtils.isManaged(file);
}
        };
    }

    private void addLocallyNewFile (Map<String, Integer> locallyNewFolders, String path) {
        if (path == null) {
            return;
        }
        boolean toAdd = true;
        String toRemove = null;
        Integer val = locallyNewFolders.get(path);
        if (val != null) {
            locallyNewFolders.put(path, val + 1);
            return;
        }
        for (Map.Entry<String, Integer> e : locallyNewFolders.entrySet()) {
            if (path.startsWith(e.getKey() + File.separator)) {
                e.setValue(e.getValue() + 1);
                toAdd = false;
                break;
            } else if (e.getKey().startsWith(path + File.separator)) {
                toRemove = e.getKey();
                break;
            }
        }
        if (toRemove != null) {
            locallyNewFolders.put(path, locallyNewFolders.remove(toRemove));
        } else if (toAdd) {
            locallyNewFolders.put(path, 1);
        }
    }

    private void addModifiedFile (Map<String, Integer> modifiedFolders, File file) {
        if (logModifiedFiles) {
        File topmost = Subversion.getInstance().getTopmostManagedAncestor(file);
        if (topmost != null) {
            String path = topmost.getAbsolutePath();
            Integer val = modifiedFolders.get(path);
            if (val == null) {
                modifiedFolders.put(path, 1);
            } else {
                modifiedFolders.put(path, val + 1);
            }
        }
        }
    }

    @NbBundle.Messages({
        "# {0} - number of changes", "# {1} - the biggest unversioned folders",
        "MSG_FileStatusCache.cacheTooBig.newFiles.text=Subversion cache contains {0} locally new (uncommitted) files. "
            + "That many uncommitted files may cause performance problems when accessing the working copy. "
            + "You should consider committing or permanently ignoring these files. "
            + "Candidates for ignoring are: {1}",
        "# {0} - folder path", "# {1} - number of contained unversioned files",
        "MSG_FileStatusCache.cacheTooBig.ignoreCandidate={0}: {1} new files"
    })
    private void logTooManyNewFiles (Map<String, Integer> locallyNewFolders, int locallyNewFiles) {
        Map<Integer, List<String>> sortedFolders = sortFolders(locallyNewFolders);
        List<String> biggestFolders = new ArrayList<String>(3);
        outer: for (Map.Entry<Integer, List<String>> e : sortedFolders.entrySet()) {
            for (String folder : e.getValue()) {
                biggestFolders.add(Bundle.MSG_FileStatusCache_cacheTooBig_ignoreCandidate(folder, e.getKey()));
                if (biggestFolders.size() == 3) {
                    break outer;
                }
            }
        }
        LOG.log(Level.WARNING, Bundle.MSG_FileStatusCache_cacheTooBig_newFiles_text(locallyNewFiles, biggestFolders));
    }

    @NbBundle.Messages({
        "# {0} - number of changes", "# {1} - checkouts with the highest number of modifications",
        "MSG_FileStatusCache.cacheTooBig.text=Subversion cache contains {0} locally modified files. "
            + "That many uncommitted files may cause performance problems when accessing the working copy. "
            + "You should consider committing or reverting these changes. "
            + "Checkouts: {1}",
        "# {0} - number of changes",
        "MSG_FileStatusCache.cacheTooBig.text.unknownFiles=Subversion cache contains {0} locally modified files. "
            + "That many uncommitted files may cause performance problems when accessing the working copy. "
            + "Run the IDE with -J-Dversioning.subversion.turbo.logModifiedFiles=true to know the exact checkout causing the problem.",
        "# {0} - checkout folder", "# {1} - number of contained modified files",
        "MSG_FileStatusCache.cacheTooBig.checkoutWithModifications={0}: {1} modifications"
    })
    private void logTooManyModifications (Map<String, Integer> modifiedFolders, int modifiedFiles) {
        Map<Integer, List<String>> sortedFolders = sortFolders(modifiedFolders);
        List<String> biggestFolders = new ArrayList<String>(3);
        outer: for (Map.Entry<Integer, List<String>> e : sortedFolders.entrySet()) {
            for (String folder : e.getValue()) {
                biggestFolders.add(Bundle.MSG_FileStatusCache_cacheTooBig_checkoutWithModifications(folder, e.getKey()));
                if (biggestFolders.size() == 3) {
                    break outer;
                }
            }
        }
        if (logModifiedFiles) {
            LOG.log(Level.WARNING, Bundle.MSG_FileStatusCache_cacheTooBig_text(modifiedFiles, biggestFolders));
        } else {
            LOG.log(Level.WARNING, Bundle.MSG_FileStatusCache_cacheTooBig_text_unknownFiles(modifiedFiles));
        }
    }
    
    private static Map<Integer, List<String>> sortFolders (Map<String, Integer> unsortedFolders) {
        Map<Integer, List<String>> sortedFolders = new TreeMap<Integer, List<String>>(new Comparator<Integer>() {
            @Override
            public int compare (Integer o1, Integer o2) {
                return - o1.compareTo(o2);
            }
        });
        for (Map.Entry<String, Integer> e : unsortedFolders.entrySet()) {
            List<String> folders = sortedFolders.get(e.getValue());
            if (folders == null) {
                sortedFolders.put(e.getValue(), folders = new ArrayList<String>());
            }
            folders.add(e.getKey());
        }
        return sortedFolders;
    }

}
