/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.write;

import SevenZip.LzmaAlone;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import thredds.featurecollection.FeatureCollectionConfig;
import ucar.nc2.grib.GribData;
import ucar.nc2.grib.grib1.Grib1RecordScanner;
import ucar.nc2.grib.grib2.Grib2Record;
import ucar.nc2.grib.grib2.Grib2RecordScanner;
import ucar.nc2.grib.grib2.Grib2SectionDataRepresentation;
import ucar.nc2.grib.grib2.Grib2Variable;
import ucar.nc2.grib.grib2.table.Grib2Tables;
import ucar.nc2.util.Misc;
import ucar.unidata.io.RandomAccessFile;
import ucar.unidata.util.test.TestDir;
import java.io.*;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Formatter;
import java.util.List;
import java.util.zip.Deflater;
import java.util.zip.Inflater;

/**
 * Try different compress algorithms on GRIB files
 *
 * @author caron
 * @since 8/12/2014
 */
public class TestGribCompressByBit {
  private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

  enum Action {
    floats, floatShaved, rawInts
  }
  enum ExtraAction {
    entropyB, entropyI
  }
  enum Algorithm {
    deflate, bzip2, bzip2T, bzipScaled, zip7
  }

  static File outDir;
  static int deflate_level = 3;
  static PrintStream detailOut;
  static PrintStream summaryOut;
  static final int showEvery = 100;

  List<CompressAlgo> runAlg = new ArrayList<>();
  Action action;
  boolean doEntropy, doEntropyI;

  TestGribCompressByBit(Action act, ExtraAction[] aa, Algorithm... wantAlgs) {
    this.action = act;
    List<ExtraAction> extra = Arrays.asList(aa);
    doEntropy = extra.contains(ExtraAction.entropyB);
    doEntropyI = extra.contains(ExtraAction.entropyI);

    for (Algorithm want : wantAlgs) {
      switch (want) {
        case deflate:
          runAlg.add(new JavaDeflate());
          break;
        case bzip2:
          runAlg.add(new ApacheBzip2());
          break;
        // case bzip2T: runAlg.add( new ItadakiBzip2()); break;
        case bzipScaled:
          runAlg.add(new ScaleAndOffset());
          break;
        // case xy: runAlg.add( new TukaaniLZMA2()); break;
        case zip7:
          runAlg.add(new Zip7());
          break;
      }
    }
    makeSummaryHeader();
  }

  int nfiles = 0;
  int tot_nrecords = 0;
  double file_gribsize;
  long file_gribread;

  void reset() {
    file_gribread = 0;
    file_gribsize = 0.0;
    for (CompressAlgo alg : runAlg)
      alg.reset();
  }

  private int doGrib2(boolean showDetail, String filename) {
    reset();
    int nrecords = 0;
    int npoints = 0;

    try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) {
      Grib2RecordScanner reader = new Grib2RecordScanner(raf);
      while (reader.hasNext()) {
        ucar.nc2.grib.grib2.Grib2Record gr = reader.next();

        Grib2SectionDataRepresentation drss = gr.getDataRepresentationSection();
        int template = drss.getDataTemplate();
        if (template != 40)
          continue; // skip all but JPEG-2000

        GribData.Info info = gr.getBinaryDataInfo(raf);
        int nBits = info.numberOfBits;
        if (nBits == 0)
          continue; // skip constant fields

        long start2 = System.nanoTime();
        float[] fdata = gr.readData(raf);
        long end2 = System.nanoTime();
        long gribReadTime = end2 - start2; // LOOK this include IO, can we just test uncompress ??
        if (npoints == 0) {
          npoints = fdata.length;
          makeDetailHeader(showDetail, filename, action, fdata.length);
        }

        int[] rawData = null;
        byte[] bdata;
        if (action == Action.rawInts) {
          rawData = gr.readRawData(raf);
          // assert rawData.length == npoints;
          bdata = GribData.convertToBytes(rawData);
        } else if (action == Action.floatShaved) {
          bdata = shaveToBytes(fdata, nBits);
        } else {
          bdata = GribData.convertToBytes(fdata);
        }

        // (int nbits, int dataPoints, int gribMsgLength, long gribReadTime, Bean bean, byte[] bdata, int[] rawData,
        // Grib2Record gr) throws IOException {

        makeDetailReport(nBits, info.ndataPoints, info.bitmapLength, info.msgLength, gribReadTime,
            new Bean(info, fdata), bdata, rawData, gr);

        nrecords++;
        tot_nrecords++;
        file_gribsize += info.msgLength;
        file_gribread += gribReadTime;
      }
    } catch (IOException e) {
      e.printStackTrace();
    }

    if (nrecords > 0)
      summaryReport(filename, npoints, nrecords);

    nfiles++;
    return nrecords;
  }

  private int doGrib1(boolean showDetail, String filename) {
    reset();
    int nrecords = 0;
    int npoints = 0;

    try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) {
      Grib1RecordScanner reader = new Grib1RecordScanner(raf);
      while (reader.hasNext()) {
        ucar.nc2.grib.grib1.Grib1Record gr = reader.next();

        // Grib2SectionDataRepresentation drss = gr.getDataRepresentationSection();
        // int template = drss.getDataTemplate();
        // if (template != 40) continue; // skip all but JPEG-2000

        GribData.Info info = gr.getBinaryDataInfo(raf);
        int nBits = info.numberOfBits;
        if (nBits == 0)
          continue; // skip constant fields

        long start2 = System.nanoTime();
        float[] fdata = gr.readData(raf);
        long end2 = System.nanoTime();
        long gribReadTime = end2 - start2; // LOOK this include IO, can we just test uncompress ??
        if (npoints == 0) {
          npoints = fdata.length;
          makeDetailHeader(showDetail, filename, action, fdata.length);
        }

        int[] rawData = null;
        byte[] bdata;
        if (action == Action.rawInts) {
          rawData = gr.readRawData(raf);
          bdata = GribData.convertToBytes(rawData);
        } else if (action == Action.floatShaved) {
          bdata = shaveToBytes(fdata, nBits);
        } else {
          bdata = GribData.convertToBytes(fdata);
        }

        // (int nbits, int dataPoints, int gribMsgLength, long gribReadTime, Bean bean, byte[] bdata, int[] rawData,
        // Grib2Record gr) throws IOException {

        makeDetailReport(nBits, info.ndataPoints, info.bitmapLength, info.msgLength, gribReadTime,
            new Bean(info, fdata), bdata, rawData, null);

        nrecords++;
        tot_nrecords++;
        file_gribsize += info.msgLength;
        file_gribread += gribReadTime;
      }
    } catch (IOException e) {
      e.printStackTrace();
    }

    if (nrecords > 0)
      summaryReport(filename, npoints, nrecords);

    nfiles++;
    return nrecords;
  }

  private void makeDetailHeader(boolean showDetail, String filename, Action action, int npoints)
      throws FileNotFoundException {
    if (showDetail) {
      File f = new File(filename);
      detailOut = new PrintStream(new File(outDir, f.getName() + ".csv"));
    } else
      detailOut = null;

    Formatter f = new Formatter();

    // first header
    f.format("%s,%s,%d%n", filename, action, npoints);

    // second header has the algo names
    f.format(",,,,,");
    if (doEntropyI)
      f.format(", ");
    if (doEntropy)
      f.format(", ");
    for (CompressAlgo alg : runAlg) {
      f.format("%s,,,,", alg.getAlgo());
    }
    f.format("%n");

    // second header
    f.format("%s, %s, %s, %s, %s, ", "nbits", "dataPoints", "bitmapLength", "gribMsgLength", "gribReadTime");
    if (doEntropyI)
      f.format("%s, ", "entropyI");
    if (doEntropy)
      f.format("%s, ", "entropyB");
    for (CompressAlgo alg : runAlg) {
      f.format("%s, %s, %s, %s, ", "compressSize", "ratio", "compressMsecs", "uncompressMsecs");
    }
    f.format("%n");

    // results
    if (detailOut != null)
      detailOut.printf("%s", f.toString());
    System.out.printf("%s", f.toString());
  }

  private void makeDetailReport(int nbits, int dataPoints, int bitmapLength, long gribMsgLength, long gribReadTime,
      Bean bean, byte[] bdata, int[] rawData, Grib2Record gr2) throws IOException {
    Formatter f = new Formatter();

    double entropyI = doEntropyI && rawData != null ? GribData.entropy(nbits, rawData) : 0.0;
    if (gr2 != null && Double.isNaN(entropyI))
      showData(gr2, bean, rawData); // debug

    double entropyB = doEntropy ? GribData.entropy(bdata) : 0.0;
    f.format("%d, %d, %d, %d, %d, ", nbits, dataPoints, bitmapLength, gribMsgLength, gribReadTime / 1000 / 1000);
    if (doEntropyI)
      f.format("%f, ", entropyI);
    if (doEntropy)
      f.format("%f, ", entropyB);

    for (CompressAlgo alg : runAlg) {
      if (alg instanceof ScaleAndOffset)
        alg.testScaleAndOffset(bean, bdata);
      else
        alg.test(bdata);
      double ratio = ((double) alg.compressSize) / gribMsgLength;
      f.format("%d, %f, %d, %d, ", alg.compressSize, ratio, alg.nsecsCompress / 1000 / 1000,
          alg.nsecsUncompress / 1000 / 1000);
    }
    f.format("%n");

    // results
    if (detailOut != null)
      detailOut.printf("%s", f.toString());
    if (tot_nrecords % showEvery == 0) {
      long snap = System.nanoTime();
      long took = (prev == 0) ? 0 : (snap - prev) / showEvery / 1000 / 1000;
      System.out.printf("%d msecs/record == %s", took, f.toString());
      prev = snap;
    }
  }

  private long prev = 0;

  private void showData(Grib2Record gr, Bean bean, int[] rawData) throws IOException {
    Grib2Tables cust = Grib2Tables.factory(gr);
    Grib2Variable gv =
        new Grib2Variable(cust, gr, 0, FeatureCollectionConfig.intvMergeDef, FeatureCollectionConfig.useGenTypeDef);
    int id = gv.hashCode();
    System.out.printf("Grib2 record cdmHash=%d (0x%s) pos=%d%n", id, Integer.toHexString(id), gr.getIs().getStartPos());

    float[] fdata = bean.readData();
    for (int i = 0; i < rawData.length; i++) {
      float convert = bean.info.convert(rawData[i]);
      if (!Misc.nearlyEquals(convert, fdata[i]))
        System.out.printf("%d %d %f (%f)%n", i, rawData[i], fdata[i], convert);
      if (rawData[i] < 0) {
        System.out.printf("*** %d %d %f (%f)%n", i, rawData[i], fdata[i], bean.info.convert(rawData[i]));
      }
    }
  }

  private void makeSummaryHeader() {
    Formatter f = new Formatter();

    // first header
    f.format("%s%n", action);

    // second header has the algo names
    f.format(",,,,,");
    for (CompressAlgo alg : runAlg) {
      f.format("%s,,,,", alg.getAlgo());
    }
    f.format("%n");

    // third header
    f.format("%s, %s, %s, %s, %s, ", "file", "npoints", "nrecords", "gribFileSize", "avg Grib read");
    for (CompressAlgo alg : runAlg) {
      f.format("%s, %s, %s, %s, ", "compressSize", "ratio", "avg compress msecs", "avg uncompress msecs");
    }
    f.format("%n");

    // results
    if (summaryOut != null)
      summaryOut.printf("%s", f.toString());
  }

  private void summaryReport(String filename, int npoints, int nrecords) {
    // double took = ((double)System.nanoTime() - start) * 1.0e-9;

    Formatter f = new Formatter();
    f.format("%s, %d, %d, %f, %f, ", filename, npoints, nrecords, file_gribsize, ((float) file_gribread) / nrecords);

    for (CompressAlgo alg : runAlg) {
      double ratio = ((double) alg.tot_size) / file_gribsize;
      double avg_compress_msecs = ((double) alg.tot_compressTime) * 1.0e-6 / nrecords;
      double avg_uncompress_msecs = ((double) alg.tot_uncompressTime) * 1.0e-6 / nrecords;
      f.format("%d, %f, %f, %f, ", alg.tot_size, ratio, avg_compress_msecs, avg_uncompress_msecs);
    }
    f.format("%n");

    System.out.printf("%s%n", f.toString());
    summaryOut.printf("%s", f.toString());
    summaryOut.flush();
  }

  class Bean implements GribData.Bean {
    GribData.Info info;
    float[] data;
    double minimum, maximum, scale;

    Bean(GribData.Info info, float[] data) {
      this.info = info;
      this.data = data;

      double pow10 = Math.pow(10.0, -getDecScale()); // 1/10^D
      minimum = (float) (pow10 * info.referenceValue); // R / 10^D
      scale = (float) (pow10 * Math.pow(2.0, getBinScale())); // 2^E / 10^D

      double maxPacked = Math.pow(2.0, getNBits()) - 1;
      maximum = minimum + scale * maxPacked;
    }

    @Override
    public float[] readData() throws IOException {
      return data;
    }

    @Override
    public int getNBits() {
      return info.numberOfBits;
    }

    @Override
    public long getDataLength() {
      return info.dataLength;
    }

    @Override
    public long getMsgLength() {
      return info.msgLength;
    }

    @Override
    public int getBinScale() {
      return info.binaryScaleFactor;
    }

    @Override
    public int getDecScale() {
      return info.decimalScaleFactor;
    }

    @Override
    public double getMinimum() {
      return minimum;
    }

    @Override
    public double getMaximum() {
      return maximum;
    }

    @Override
    public double getScale() {
      return scale;
    }
  }

  //////////////////////////////////////////////////////////////////////
  abstract class CompressAlgo {
    long nsecsCompress, nsecsUncompress;
    int orgSize, compressSize, uncompressSize;

    long tot_size;
    long tot_compressTime, tot_uncompressTime;

    void test(byte[] orgBytes) throws IOException {
      orgSize = orgBytes.length;

      long start = System.nanoTime();
      byte[] compressedBytes = compress(orgBytes);
      long end = System.nanoTime();
      nsecsCompress = end - start;
      compressSize = compressedBytes.length;

      start = end;
      byte[] uncompressedBytes = uncompress(compressedBytes);
      end = System.nanoTime();
      nsecsUncompress = end - start;
      uncompressSize = uncompressedBytes.length;
      /*
       * if (uncompressSize != orgSize) {
       * System.out.printf("HEY %s %d != %d%n", getAlgo(), uncompressSize, orgSize);
       * } /* else {
       * Formatter f = new Formatter();
       * if (!Misc.compare(orgBytes, uncompressedBytes, f))
       * System.out.printf("%s%n", f);
       * }
       */

      tot_size += compressSize;
      tot_compressTime += nsecsCompress;
      tot_uncompressTime += nsecsUncompress;
    }

    void testScaleAndOffset(GribData.Bean bean, byte[] orgBytes) throws IOException {
      orgSize = orgBytes.length;

      ScaleAndOffset scaler = (ScaleAndOffset) this;

      long start = System.nanoTime();
      byte[] compressedBytes = scaler.compress(bean);
      long end = System.nanoTime();
      nsecsCompress = end - start;
      compressSize = compressedBytes.length;

      start = end;
      byte[] uncompressedBytes = uncompress(compressedBytes);
      end = System.nanoTime();
      nsecsUncompress = end - start;
      uncompressSize = uncompressedBytes.length;

      tot_size += compressSize;
      tot_compressTime += nsecsCompress;
      tot_uncompressTime += nsecsUncompress;
    }


    void reset() {
      this.tot_size = 0;
      this.tot_compressTime = 0;
      this.tot_uncompressTime = 0;
    }

    abstract byte[] compress(byte[] data) throws IOException;

    abstract byte[] uncompress(byte[] data) throws IOException;

    abstract Algorithm getAlgo();
  }

  class JavaDeflate extends CompressAlgo {
    Algorithm getAlgo() {
      return Algorithm.deflate;
    }

    byte[] compress(byte[] data) {
      Deflater compresser = new Deflater(deflate_level);
      compresser.setInput(data);
      compresser.finish();

      byte[] output = new byte[data.length * 2];
      int size = compresser.deflate(output);
      compresser.end();

      // copy just the resulting bytes
      byte[] out = new byte[size];
      System.arraycopy(output, 0, out, 0, size);
      return out;
    }

    byte[] uncompress(byte[] data) {
      byte[] result = new byte[data.length * 20];
      int resultLength;
      try {
        Inflater decompresser = new Inflater();
        decompresser.setInput(data, 0, data.length);
        resultLength = decompresser.inflate(result);
        decompresser.end();

      } catch (Exception e) {
        e.printStackTrace();
        return new byte[0];
      }

      // copy just the resulting bytes
      byte[] out = new byte[resultLength];
      System.arraycopy(result, 0, out, 0, resultLength);
      return out;
    }

  }

  public static byte[] shaveToBytes(float[] data, int bits) {
    int bitMask = GribToNetcdfWriter.getBitMask(bits + 1);
    ByteBuffer bb = ByteBuffer.allocate(data.length * 4);
    for (float d : data)
      bb.putFloat(GribToNetcdfWriter.bitShave(d, bitMask));
    return bb.array();
  }

  public static byte[] shavePrecision(float[] org_data, int bits) {
    double expectedPrecision = Math.pow(2.0, -(bits + 1));

    int bitMask = GribToNetcdfWriter.getBitMask(bits + 1);
    ByteBuffer bb = ByteBuffer.allocate(org_data.length * 4);
    for (float d : org_data)
      bb.putFloat(GribToNetcdfWriter.bitShave(d, bitMask));

    int count = 0;
    float[] result = new float[org_data.length];
    bb.flip();
    float max_diff = -Float.MAX_VALUE;
    float max_pdiff = -Float.MAX_VALUE;
    for (float d : org_data) {
      float shaved = bb.getFloat();
      result[count++] = shaved;
      float diff = Math.abs(d - shaved);
      float pdiff = (d != 0.0) ? diff / d : 0.0f;
      max_diff = Math.max(max_diff, diff);
      max_pdiff = Math.max(max_pdiff, pdiff);
    }
    if (max_pdiff != 0.0) {
      if (max_pdiff < expectedPrecision / 2 || max_pdiff > expectedPrecision)
        System.out.printf("nbits=%d diff=%f pdiff=%g expect=%g%n", bits, max_diff, max_pdiff, expectedPrecision);
    }

    return bb.array();
  }

  public static void checkShavedPrecision(float[] org, int bits) {
    double expectedPrecision = Math.pow(2.0, -bits);
    float max_pdiff = -Float.MAX_VALUE;
    int bitMask = GribToNetcdfWriter.getBitMask(bits);
    for (int i = 0; i < org.length; i++) {
      float d = org[i];
      float shaved = GribToNetcdfWriter.bitShave(org[i], bitMask);
      float diff = Math.abs(d - shaved);
      float pdiff = (d != 0.0) ? diff / d : 0.0f;
      assert pdiff < expectedPrecision;
      max_pdiff = Math.max(max_pdiff, pdiff);
    }

    if (max_pdiff != 0.0) { // usually max precision lies between 1/2^N and 1/2^(N+1)
      if (max_pdiff < expectedPrecision / 2 || max_pdiff > expectedPrecision)
        System.out.printf("nbits=%d max_pdiff=%g expect=%g%n", bits, max_pdiff, expectedPrecision);
    }
  }

  class ApacheBzip2 extends CompressAlgo {
    Algorithm getAlgo() {
      return Algorithm.bzip2;
    }

    byte[] compress(byte[] bdata) {
      ByteArrayOutputStream out = new ByteArrayOutputStream(bdata.length);
      try (BZip2CompressorOutputStream bzOut = new BZip2CompressorOutputStream(out)) {
        bzOut.write(bdata, 0, bdata.length);
        bzOut.finish();

      } catch (Exception e) {
        e.printStackTrace();
      }

      return out.toByteArray();
    }

    byte[] uncompress(byte[] bdata) {
      ByteArrayOutputStream out = new ByteArrayOutputStream(20 * bdata.length);
      ByteArrayInputStream in = new ByteArrayInputStream(bdata);
      try (BZip2CompressorInputStream bzIn = new BZip2CompressorInputStream(in, false)) {
        int bytesRead;
        byte[] decoded = new byte[524288];
        while ((bytesRead = bzIn.read(decoded)) != -1) {
          out.write(decoded, 0, bytesRead);
        }
        out.close();

      } catch (Exception e) {
        e.printStackTrace();
      }

      return out.toByteArray();
    }
  }

  class ScaleAndOffset extends CompressAlgo {
    @Override
    byte[] compress(byte[] data) throws IOException {
      return new byte[0];
    }

    @Override
    byte[] uncompress(byte[] data) throws IOException {
      float[] result = GribData.uncompressScaled(data);
      return GribData.convertToBytes(result);
    }

    Algorithm getAlgo() {
      return Algorithm.bzipScaled;
    }

    byte[] compress(GribData.Bean bean) throws IOException {
      return GribData.compressScaled(bean); // LOOK could separate from compression and try different ones
    }
  }

  /*
   * class TukaaniLZMA2 extends CompressAlgo {
   * Algorithm getAlgo() {
   * return Algorithm.xy;
   * }
   * 
   * byte[] compress(byte[] bdata) throws IOException {
   * ByteArrayOutputStream out = new ByteArrayOutputStream(bdata.length * 2);
   * try (XZOutputStream compressor = new XZOutputStream(out, new LZMA2Options())) {
   * compressor.write(bdata, 0, bdata.length);
   * compressor.finish();
   * } catch (IOException e) {
   * e.printStackTrace();
   * }
   * 
   * return out.toByteArray();
   * }
   * 
   * byte[] uncompress(byte[] bdata) {
   * ByteArrayOutputStream out = new ByteArrayOutputStream(20 * bdata.length);
   * ByteArrayInputStream in = new ByteArrayInputStream(bdata);
   * try (XZInputStream bzIn = new XZInputStream(in)) {
   * int bytesRead;
   * byte[] decoded = new byte [524288];
   * while ((bytesRead = bzIn.read (decoded)) != -1) {
   * out.write (decoded, 0, bytesRead) ;
   * }
   * out.close();
   * 
   * } catch (Exception e) {
   * e.printStackTrace();
   * }
   * 
   * return out.toByteArray();
   * }
   * 
   * }
   */

  class Zip7 extends CompressAlgo {
    Algorithm getAlgo() {
      return Algorithm.zip7;
    }

    byte[] compress(byte[] bdata) throws IOException {
      ByteArrayOutputStream outStream = new ByteArrayOutputStream(bdata.length * 2);
      InputStream inStream = new ByteArrayInputStream(bdata);

      // the compression params
      LzmaAlone.CommandLine params = new LzmaAlone.CommandLine();
      SevenZip.Compression.LZMA.Encoder encoder = new SevenZip.Compression.LZMA.Encoder();
      if (!encoder.SetAlgorithm(params.Algorithm))
        throw new IOException("Incorrect compression mode");
      if (!encoder.SetDictionarySize(params.DictionarySize))
        throw new IOException("Incorrect dictionary size");
      if (!encoder.SetNumFastBytes(params.Fb))
        throw new IOException("Incorrect -fb value");
      if (!encoder.SetMatchFinder(params.MatchFinder))
        throw new IOException("Incorrect -mf value");
      if (!encoder.SetLcLpPb(params.Lc, params.Lp, params.Pb))
        throw new IOException("Incorrect -lc or -lp or -pb value");

      encoder.SetEndMarkerMode(false);
      encoder.WriteCoderProperties(outStream);

      // next 8 bytes are the length
      long msgSize = (long) bdata.length;
      for (int i = 0; i < 8; i++)
        outStream.write((int) (msgSize >>> (8 * i)) & 0xFF);

      // encode the data
      encoder.Code(inStream, outStream, -1, -1, null);

      return outStream.toByteArray();
    }

    byte[] uncompress(byte[] bdata) throws IOException {
      ByteArrayOutputStream outStream = new ByteArrayOutputStream(bdata.length * 20);
      InputStream inStream = new ByteArrayInputStream(bdata);

      // the compression params
      int propertiesSize = 5;
      byte[] properties = new byte[propertiesSize];
      if (inStream.read(properties, 0, propertiesSize) != propertiesSize)
        throw new IOException("input .lzma file is too short");
      SevenZip.Compression.LZMA.Decoder decoder = new SevenZip.Compression.LZMA.Decoder();
      if (!decoder.SetDecoderProperties(properties))
        throw new IOException("Incorrect stream properties");

      // next 8 bytes are the length
      long outSize = 0;
      for (int i = 0; i < 8; i++) {
        int v = inStream.read();
        if (v < 0)
          throw new IOException("Can't read stream size");
        outSize |= ((long) v) << (8 * i);
      }

      // decode the data
      if (!decoder.Code(inStream, outStream, outSize))
        throw new IOException("Error in data stream");

      return outStream.toByteArray();
    }

  }

  ///////////////////////////////////////////////////////


  /*
   * private int doGrib1(String filename) {
   * try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) {
   * Grib1RecordScanner reader = new Grib1RecordScanner(raf);
   * while (reader.hasNext()) {
   * ucar.nc2.grib.grib1.Grib1Record gr = reader.next();
   * 
   * Grib1SectionBinaryData dataSection = gr.getDataSection();
   * Grib1SectionBinaryData.BinaryDataInfo info = dataSection.getBinaryDataInfo(raf);
   * 
   * float[] data = gr.readData(raf);
   * int compressDeflate = deflate( convertToBytes(data)); // run it through the deflator
   * if (npoints == 0) npoints = data.length;
   * 
   * // deflate the original (bit-packed) data
   * int compressOrg = deflate(dataSection.getBytes(raf)); // run it through the deflator
   * 
   * // results
   * if (detailOut != null) detailOut.printf("%d, %d, %d, %d, %d%n", info.numbits, data.length, info.msgLength,
   * compressDeflate, compressOrg);
   * //System.out.printf("%d, %d, %d, %d, %f, %d%n", info.numbits, data.length, info.msgLength, compressBytes, r,
   * compressOrg);
   * nrecords++;
   * totalBytesIn += info.msgLength;
   * totDeflate += compressDeflate;
   * totOrg += compressOrg;
   * }
   * } catch (IOException e) {
   * e.printStackTrace();
   * }
   * return nrecords;
   * }
   */


  private static class CompressReportAction implements TestDir.Act {
    TestGribCompressByBit compressByBit;
    boolean showDetail;

    private CompressReportAction(TestGribCompressByBit compressByBit, boolean showDetail) {
      this.compressByBit = compressByBit;
      this.showDetail = showDetail;
    }

    public int doAct(String filename) throws IOException {
      if (filename.endsWith("grib1"))
        compressByBit.doGrib1(showDetail, filename);
      else
        compressByBit.doGrib2(showDetail, filename);

      if (detailOut != null)
        detailOut.close();

      return 1;
    }
  }


  public static void main(String[] args) throws IOException {
    outDir = new File("E:/grib2nc/ecmwf-ints/");
    outDir.mkdirs();
    summaryOut = new PrintStream(new File(outDir, "summary.csv"));

    try {
      ExtraAction[] extras = new ExtraAction[] {};
      TestGribCompressByBit compressByBit =
          new TestGribCompressByBit(Action.rawInts, extras, Algorithm.deflate, Algorithm.bzip2T, Algorithm.zip7);
      CompressReportAction test = new CompressReportAction(compressByBit, true);
      test.doAct("E:/work/ecmwf/oper.sfc.grib1");

    } finally {
      summaryOut.close();
    }
  }

  public static void main2(String[] args) throws IOException {
    outDir = new File("E:/grib2nc/all-ints/");
    outDir.mkdirs();
    summaryOut = new PrintStream(new File(outDir, "summary.csv"));

    try {
      ExtraAction[] extras = new ExtraAction[] {};
      TestGribCompressByBit compressByBit =
          new TestGribCompressByBit(Action.rawInts, extras, Algorithm.deflate, Algorithm.bzip2T, Algorithm.zip7);
      CompressReportAction test = new CompressReportAction(compressByBit, true);
      String dirName = "Q:/cdmUnitTest/tds/ncep/";
      TestDir.actOnAll(dirName, new MyFileFilter(), test);

    } finally {
      summaryOut.close();
    }
  }

  private static class MyFileFilter implements FileFilter {

    @Override
    public boolean accept(File pathname) {
      String name = pathname.getName();
      if (name.contains("GEFS_Global_1p0deg_Ensemble"))
        return false; // too big
      return name.endsWith(".grib2");
    }
  }

  /*
   * public static void main2(String[] args) throws IOException {
   * outDir = new File("G:/grib2nc/bzip2/");
   * summaryOut = new PrintStream(new File(outDir, "summary.csv"));
   * writeSummaryHeader();
   * 
   * try {
   * String dirName = "Q:/cdmUnitTest/tds/ncep/";
   * TestDir.actOnAll(dirName, new TestDir.FileFilterFromSuffixes("grib2"), new CompressReportAction(true, false),
   * false);
   * 
   * } finally {
   * summaryOut.close();
   * }
   * }
   */

}

/*
 * 
 * GFS_Alaska_191km_20100913_0000.grib1
 * GFS_CONUS_191km_20100519_1800.grib1
 * GFS_CONUS_80km_20100513_0600.grib1
 * GFS_CONUS_95km_20100506_0600.grib1
 * GFS_Hawaii_160km_20100428_0000.grib1
 * GFS_N_Hemisphere_381km_20100516_0600.grib1
 * GFS_Puerto_Rico_191km_20100515_0000.grib1
 * NAM_Alaska_22km_20100504_0000.grib1
 * NAM_Alaska_45km_noaaport_20100525_1200.grib1
 * NAM_Alaska_95km_20100502_0000.grib1
 * NAM_CONUS_20km_noaaport_20100602_0000.grib1
 * NAM_CONUS_80km_20100508_1200.grib1
 * RUC2_CONUS_40km_20100515_0200.grib1
 * RUC2_CONUS_40km_20100914_1200.grib1
 * RUC_CONUS_80km_20100430_0000.grib1
 * 
 * -----------------
 * D = has duplicates
 * 
 * DRS template
 * 0: count = 30
 * 2: count = 283
 * 3: count = 15017
 * 40: count = 384980
 * 
 * 
 * DGEX_Alaska_12km_20100524_0000.grib2
 * DGEX_CONUS_12km_20100514_1800.grib2
 * GEFS_Global_1p0deg_Ensemble_20120215_0000.grib2
 * GEFS_Global_1p0deg_Ensemble_derived_20120214_0000.grib2
 * GFS_Global_0p5deg_20100913_0000.grib2
 * GFS_Global_0p5deg_20140804_0000.grib2
 * 3 GFS_Global_2p5deg_20100602_1200.grib2
 * GFS_Global_onedeg_20100913_0000.grib2
 * GFS_Puerto_Rico_0p5deg_20140106_1800.grib2
 * 3 HRRR_CONUS_3km_wrfprs_201408120000.grib2
 * NAM_Alaska_11km_20100519_0000.grib2
 * NAM_Alaska_45km_conduit_20100913_0000.grib2
 * NAM_CONUS_12km_20100915_1200.grib2
 * NAM_CONUS_12km_20140804_0000.grib2
 * NAM_CONUS_12km_conduit_20140804_0000.grib2
 * NAM_CONUS_20km_selectsurface_20100913_0000.grib2
 * NAM_CONUS_20km_surface_20100913_0000.grib2
 * NAM_CONUS_40km_conduit_20100913_0000.grib2
 * NAM_Firewxnest_20140804_0000.grib2
 * NAM_Polar_90km_20100913_0000.grib2
 * 2 NDFD_CONUS_5km_20140805_1200.grib2
 * 2/3 NDFD_CONUS_5km_conduit_20140804_0000.grib2
 * 2 NDFD_Fireweather_CONUS_20140804_1800.grib2
 * RR_CONUS_13km_20121028_0000.grib2
 * RR_CONUS_20km_20140804_1900.grib2
 * RR_CONUS_40km_20140805_1600.grib2
 * 0/2 RTMA_CONUS_2p5km_20111221_0800.grib2
 * 0 RTMA_GUAM_2p5km_20140803_0600.grib2
 * RUC2_CONUS_20km_hybrid_20100913_0000.grib2
 * RUC2_CONUS_20km_pressure_20100509_1300.grib2
 * RUC2_CONUS_20km_surface_20100516_1600.grib2
 * SREF_Alaska_45km_ensprod_20120213_1500.grib2
 * SREF_CONUS_40km_ensprod_20120214_1500.grib2
 * SREF_CONUS_40km_ensprod_20140804_1500.grib2
 * SREF_CONUS_40km_ensprod_biasc_20120213_2100.grib2
 * D SREF_CONUS_40km_ensprod_biasc_20140805_1500.grib2
 * SREF_CONUS_40km_pgrb_biasc_nmm_n2_20100602_1500.grib2
 * SREF_CONUS_40km_pgrb_biasc_rsm_p2_20120213_1500.grib2
 * SREF_PacificNE_0p4_ensprod_20120213_2100.grib2
 * D WW3_Coastal_Alaska_20140804_0000.grib2
 * D WW3_Coastal_US_East_Coast_20140804_0000.grib2
 * D WW3_Coastal_US_West_Coast_20140804_1800.grib2
 * D WW3_Global_20140804_0000.grib2
 * D WW3_Regional_Alaska_20140804_0000.grib2
 * D WW3_Regional_Eastern_Pacific_20140804_0000.grib2
 * D WW3_Regional_US_East_Coast_20140804_0000.grib2
 * D WW3_Regional_US_West_Coast_20140803_0600.grib2
 * 
 * 
 */
