/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.search.facet;

import static org.apache.solr.search.facet.FacetField.FacetMethod;
import static org.apache.solr.search.facet.SlotAcc.SweepingCountSlotAcc.SWEEP_COLLECTION_DEBUG_KEY;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.embedded.JettySolrRunner;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.noggit.JSONUtil;
import org.noggit.JSONWriter;
import org.noggit.JSONWriter.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A randomized test of nested facets using the <code>relatedness()</code> function, that asserts
 * the results are consistent and equivalent regardless of what <code>method</code> (ie:
 * FacetFieldProcessor) and/or <code>{@value RelatednessAgg#SWEEP_COLLECTION}</code> option is
 * requested.
 *
 * <p>This test is based on {@link TestCloudJSONFacetSKG} but does <em>not</em> force <code>
 * refine: true</code> nor specify a <code>domain: { 'query':'*:*' }</code> for every facet, because
 * this test does not attempt to prove the results with validation requests.
 *
 * <p>This test only concerns itself with the equivalency of results
 *
 * @see TestCloudJSONFacetSKG
 */
public class TestCloudJSONFacetSKGEquiv extends SolrCloudTestCase {

  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

  private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
  private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";

  private static final int MAX_FIELD_NUM = 15;
  private static final int UNIQUE_FIELD_VALS = 50;

  /** Multi-Valued string field suffixes that can be randomized for testing diff facet code paths */
  private static final String[] MULTI_STR_FIELD_SUFFIXES =
      new String[] {"_multi_ss", "_multi_sds", "_multi_sdsS"};
  /** Multi-Valued int field suffixes that can be randomized for testing diff facet code paths */
  private static final String[] MULTI_INT_FIELD_SUFFIXES =
      new String[] {"_multi_is", "_multi_ids", "_multi_idsS"};

  /**
   * Single Valued string field suffixes that can be randomized for testing diff facet code paths
   */
  private static final String[] SOLO_STR_FIELD_SUFFIXES =
      new String[] {"_solo_s", "_solo_sd", "_solo_sdS"};
  /** Single Valued int field suffixes that can be randomized for testing diff facet code paths */
  private static final String[] SOLO_INT_FIELD_SUFFIXES =
      new String[] {"_solo_i", "_solo_id", "_solo_idS"};

  /** A basic client for operations at the cloud level, default collection will be set */
  private static CloudSolrClient CLOUD_CLIENT;
  /** One client per node */
  private static final ArrayList<SolrClient> CLIENTS = new ArrayList<>(5);

  @BeforeClass
  public static void createMiniSolrCloudCluster() throws Exception {
    // check constants
    assertTrue(
        "bad test constants: some suffixes will never be tested",
        (MULTI_STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM)
            && (MULTI_INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM)
            && (SOLO_STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM)
            && (SOLO_INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));

    // we need DVs on point fields to compute stats & facets
    if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))
      System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");

    // multi replicas should not matter...
    final int repFactor = usually() ? 1 : 2;
    // ... but we definitely want to test multiple shards
    final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 : 3));
    final int numNodes = (numShards * repFactor);

    final String configName = DEBUG_LABEL + "_config-set";
    final Path configDir = TEST_COLL1_CONF();

    configureCluster(numNodes).addConfig(configName, configDir).configure();

    Map<String, String> collectionProperties = new LinkedHashMap<>();
    collectionProperties.put("config", "solrconfig-tlog.xml");
    collectionProperties.put("schema", "schema_latest.xml");
    CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
        .setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE)
        .setProperties(collectionProperties)
        .process(cluster.getSolrClient());

    CLOUD_CLIENT = cluster.basicSolrClientBuilder().withDefaultCollection(COLLECTION_NAME).build();

    waitForRecoveriesToFinish(CLOUD_CLIENT);

    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
      CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
    }

    final int numDocs = atLeast(100);
    for (int id = 0; id < numDocs; id++) {
      SolrInputDocument doc = sdoc("id", "" + id);

      // NOTE: for each fieldNum, there are actaully 4 fields: multi(str+int) + solo(str+int)
      for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
        // NOTE: Some docs may not have any value in some fields
        final int numValsThisDoc = TestUtil.nextInt(random(), 0, (usually() ? 5 : 10));
        for (int v = 0; v < numValsThisDoc; v++) {
          final String fieldValue = randFieldValue(fieldNum);

          // multi valued: one string, and one integer
          doc.addField(multiStrField(fieldNum), fieldValue);
          doc.addField(multiIntField(fieldNum), fieldValue);
        }
        // use num values in multivalue to inform sparseness of single value
        if (3 <= numValsThisDoc) {
          final String fieldValue = randFieldValue(fieldNum);
          doc.addField(soloStrField(fieldNum), fieldValue);
          doc.addField(soloIntField(fieldNum), fieldValue);
        }
      }
      CLOUD_CLIENT.add(doc);
      if (random().nextInt(100) < 1) {
        CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
      }
      if (random().nextInt(100) < 5) {
        CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
      }
    }
    CLOUD_CLIENT.commit();

    log.info(
        "Created {} using numNodes={}, numShards={}, repFactor={}, numDocs={}",
        COLLECTION_NAME,
        numNodes,
        numShards,
        repFactor,
        numDocs);
  }

  /**
   * Given a (random) number, and a (static) array of possible suffixes returns a consistent field
   * name that uses that number and one of the specified suffixes in its name.
   *
   * @see #MULTI_STR_FIELD_SUFFIXES
   * @see #MULTI_INT_FIELD_SUFFIXES
   * @see #MAX_FIELD_NUM
   * @see #randFieldValue
   */
  private static String field(final String[] suffixes, final int fieldNum) {
    assertTrue(fieldNum < MAX_FIELD_NUM);

    final String suffix = suffixes[fieldNum % suffixes.length];
    return "field_" + fieldNum + suffix;
  }
  /** Given a (random) number, returns a consistent field name for a multivalued string field */
  private static String multiStrField(final int fieldNum) {
    return field(MULTI_STR_FIELD_SUFFIXES, fieldNum);
  }
  /** Given a (random) number, returns a consistent field name for a multivalued int field */
  private static String multiIntField(final int fieldNum) {
    return field(MULTI_INT_FIELD_SUFFIXES, fieldNum);
  }
  /** Given a (random) number, returns a consistent field name for a single valued string field */
  private static String soloStrField(final int fieldNum) {
    return field(SOLO_STR_FIELD_SUFFIXES, fieldNum);
  }
  /** Given a (random) number, returns a consistent field name for a single valued int field */
  private static String soloIntField(final int fieldNum) {
    return field(SOLO_INT_FIELD_SUFFIXES, fieldNum);
  }

  /**
   * Given a (random) field number, returns a random (integer based) value for that field. NOTE: The
   * number of unique values in each field is constant according to {@link #UNIQUE_FIELD_VALS} but
   * the precise <em>range</em> of values will vary for each unique field number, such that cross
   * field joins will match fewer documents based on how far apart the field numbers are.
   *
   * @see #UNIQUE_FIELD_VALS
   * @see #field
   */
  private static String randFieldValue(final int fieldNum) {
    return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
  }

  @AfterClass
  public static void afterClass() throws Exception {
    IOUtils.closeQuietly(CLOUD_CLIENT);
    for (SolrClient client : CLIENTS) {
      IOUtils.closeQuietly(client);
    }
    CLIENTS.clear();
  }

  /**
   * Sanity check that our method of varying the <code>method</code> param works and can be verified
   * by inspecting the debug output of basic requests.
   */
  public void testWhiteboxSanityMethodProcessorDebug() {
    // NOTE: json.facet debugging output can be wonky, particularly when dealing with cloud
    // so for these queries we keep it simple:
    // - only one "top" facet per request
    // - no refinement
    // even with those constraints in place, a single facet can (may/sometimes?) produce multiple
    // debug blocks - apparently due to shard merging? So...
    // - only inspect the "first" debug NamedList in the results
    //

    // simple individual facet that sorts on a skg stat...
    final TermFacet f = new TermFacet(soloStrField(9), 10, 0, "skg desc", null);
    final Map<String, TermFacet> facets = new LinkedHashMap<>();
    facets.put("str", f);

    final SolrParams facetParams =
        params(
            "rows", "0",
            "debug", "true", // SOLR-14451
            // *:* is the only "safe" query for this test,
            // to ensure we always have at least one bucket for every facet,
            // so we can be confident in getting the debug we expect...
            "q", "*:*",
            "fore", multiStrField(7) + ":11",
            "back", "*:*",
            "json.facet", Facet.toJSONFacetParamValue(facets));

    { // dv
      final SolrParams params = SolrParams.wrapDefaults(params("method_val", "dv"), facetParams);
      final NamedList<Object> debug = getFacetDebug(params);
      assertEquals(FacetFieldProcessorByArrayDV.class.getSimpleName(), debug.get("processor"));
    }
    { // dvhash
      final SolrParams params =
          SolrParams.wrapDefaults(params("method_val", "dvhash"), facetParams);
      final NamedList<Object> debug = getFacetDebug(params);
      assertEquals(FacetFieldProcessorByHashDV.class.getSimpleName(), debug.get("processor"));
    }
  }

  /**
   * Sanity check that our method of varying the <code>{@value RelatednessAgg#SWEEP_COLLECTION}
   * </code> in conjunction with the <code>method</code> params works and can be verified by
   * inspecting the debug output of basic requests.
   */
  public void testWhiteboxSanitySweepDebug() {
    // NOTE: json.facet debugging output can be wonky, particularly when dealing with cloud
    // so for these queries we keep it simple:
    // - only one "top" facet per request
    // - no refinement
    // even with those constraints in place, a single facet can (may/sometimes?) produce multiple
    // debug blocks - apparently due to shard merging? So...
    // - only inspect the "first" debug NamedList in the results
    //

    final SolrParams baseParams =
        params(
            "rows", "0",
            "debug", "true", // SOLR-14451
            // *:* is the only "safe" query for this test,
            // to ensure we always have at least one bucket for every facet,
            // so we can be confident in getting the debug we expect...
            "q", "*:*",
            "fore", multiStrField(7) + ":11",
            "back", "*:*");

    // simple individual facet that sorts on a skg stat...
    //
    // all results we test should be the same even if there is another 'skg_extra' stat,
    // it shouldn't be involved in the sweeping at all.
    for (Facet extra : Arrays.asList(null, new RelatednessFacet(multiStrField(2) + ":9", null))) {
      // choose a single value string, so we know both 'dv' (sweep) and 'dvhash' (no sweep) can be
      // specified
      final TermFacet f = new TermFacet(soloStrField(9), 10, 0, "skg desc", null);
      if (null != extra) {
        f.subFacets.put("skg_extra", extra);
      }
      final Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("str", f);

      final SolrParams facetParams =
          SolrParams.wrapDefaults(
              params("method_val", "dv", "json.facet", Facet.toJSONFacetParamValue(facets)),
              baseParams);

      // both default sweep option and explicit sweep should give same results...
      for (SolrParams sweepParams :
          Arrays.asList(
              params(),
              params("sweep_key", RelatednessAgg.SWEEP_COLLECTION, "sweep_val", "true"))) {
        final SolrParams params = SolrParams.wrapDefaults(sweepParams, facetParams);

        final NamedList<Object> debug = getFacetDebug(params);
        assertEquals(FacetFieldProcessorByArrayDV.class.getSimpleName(), debug.get("processor"));
        @SuppressWarnings("unchecked")
        final NamedList<Object> sweep_debug =
            (NamedList<Object>) debug.get(SWEEP_COLLECTION_DEBUG_KEY);
        assertNotNull(sweep_debug);
        assertEquals("count", sweep_debug.get("base"));
        assertEquals(Arrays.asList("skg!fg", "skg!bg"), sweep_debug.get("accs"));
        assertEquals(Arrays.asList("skg"), sweep_debug.get("mapped"));
      }
      { // 'dv' will always *try* to sweep, but disabling on stat should mean debug is mostly
        // empty...
        final SolrParams params =
            SolrParams.wrapDefaults(
                params("sweep_key", RelatednessAgg.SWEEP_COLLECTION, "sweep_val", "false"),
                facetParams);
        final NamedList<Object> debug = getFacetDebug(params);
        assertEquals(FacetFieldProcessorByArrayDV.class.getSimpleName(), debug.get("processor"));
        @SuppressWarnings("unchecked")
        final NamedList<Object> sweep_debug =
            (NamedList<Object>) debug.get(SWEEP_COLLECTION_DEBUG_KEY);
        assertNotNull(sweep_debug);
        assertEquals("count", sweep_debug.get("base"));
        assertEquals(Collections.emptyList(), sweep_debug.get("accs"));
        assertEquals(Collections.emptyList(), sweep_debug.get("mapped"));
      }
      { // if we override 'dv' with 'hashdv' which doesn't sweep, our sweep debug should be empty,
        // even if the skg stat does ask for sweeping explicitly...
        final SolrParams params =
            SolrParams.wrapDefaults(
                params(
                    "method_val", "dvhash",
                    "sweep_key", RelatednessAgg.SWEEP_COLLECTION,
                    "sweep_val", "true"),
                facetParams);
        final NamedList<Object> debug = getFacetDebug(params);
        assertEquals(FacetFieldProcessorByHashDV.class.getSimpleName(), debug.get("processor"));
        assertNull(debug.get(SWEEP_COLLECTION_DEBUG_KEY));
      }
    }

    // simple facet that sorts on a skg stat but uses prelim_sort on count
    //
    // all results we test should be the same even if there is another 'skg_extra' stat,
    // neither skg should be involved in the sweeping at all.
    for (Facet extra : Arrays.asList(null, new RelatednessFacet(multiStrField(2) + ":9", null))) {
      // choose a single value string, so we know both 'dv' (sweep) and 'dvhash' (no sweep) can be
      // specified
      final TermFacet f =
          new TermFacet(
              soloStrField(9),
              map("limit", 3, "overrequest", 0, "sort", "skg desc", "prelim_sort", "count asc"));
      if (null != extra) {
        f.subFacets.put("skg_extra", extra);
      }
      final Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("str", f);

      final SolrParams facetParams =
          SolrParams.wrapDefaults(
              params("method_val", "dv", "json.facet", Facet.toJSONFacetParamValue(facets)),
              baseParams);

      // default sweep as well as any explicit sweep=true/false values should give same results: no
      // sweeping
      for (SolrParams sweepParams :
          Arrays.asList(
              params(),
              params("sweep_key", RelatednessAgg.SWEEP_COLLECTION, "sweep_val", "false"),
              params("sweep_key", RelatednessAgg.SWEEP_COLLECTION, "sweep_val", "true"))) {
        final SolrParams params = SolrParams.wrapDefaults(sweepParams, facetParams);

        final NamedList<Object> debug = getFacetDebug(params);
        assertEquals(FacetFieldProcessorByArrayDV.class.getSimpleName(), debug.get("processor"));
        @SuppressWarnings("unchecked")
        final NamedList<Object> sweep_debug =
            (NamedList<Object>) debug.get(SWEEP_COLLECTION_DEBUG_KEY);
        assertNotNull(sweep_debug);
        assertEquals("count", sweep_debug.get("base"));
        assertEquals(Collections.emptyList(), sweep_debug.get("accs"));
        assertEquals(Collections.emptyList(), sweep_debug.get("mapped"));
      }
    }

    { // single facet with infinite limit + multiple skgs...
      // this should trigger MultiAcc collection, causing sweeping on both skg functions
      //
      // all results we test should be the same even if there is another 'min' stat,
      // in each term facet.  it shouldn't affect the sweeping/MultiAcc at all.
      for (Facet extra : Arrays.asList(null, new SumFacet(multiIntField(2)))) {
        final Map<String, TermFacet> facets = new LinkedHashMap<>();
        final TermFacet facet = new TermFacet(soloStrField(9), -1, 0, "skg2 desc", null);
        facet.subFacets.put("skg2", new RelatednessFacet(multiStrField(2) + ":9", null));
        if (null != extra) {
          facet.subFacets.put("sum", extra);
        }
        facets.put("str", facet);
        final SolrParams facetParams =
            SolrParams.wrapDefaults(
                params("method_val", "dv", "json.facet", Facet.toJSONFacetParamValue(facets)),
                baseParams);

        // both default sweep option and explicit sweep should give same results...
        for (SolrParams sweepParams :
            Arrays.asList(
                params(),
                params("sweep_key", RelatednessAgg.SWEEP_COLLECTION, "sweep_val", "true"))) {
          final SolrParams params = SolrParams.wrapDefaults(sweepParams, facetParams);

          final NamedList<Object> debug = getFacetDebug(params);
          assertEquals(FacetFieldProcessorByArrayDV.class.getSimpleName(), debug.get("processor"));
          @SuppressWarnings("unchecked")
          final NamedList<Object> sweep_debug =
              (NamedList<Object>) debug.get(SWEEP_COLLECTION_DEBUG_KEY);
          assertNotNull(sweep_debug);
          assertEquals("count", sweep_debug.get("base"));
          assertEquals(
              Arrays.asList("skg!fg", "skg!bg", "skg2!fg", "skg2!bg"), sweep_debug.get("accs"));
          assertEquals(Arrays.asList("skg", "skg2"), sweep_debug.get("mapped"));
        }
      }
    }

    // nested facets that both sort on a skg stat
    // (set limit + overrequest tiny to keep multishard response manageable)
    //
    // all results we test should be the same even if there is another 'skg_extra' stat,
    // in each term facet.  they shouldn't be involved in the sweeping at all.
    for (Facet extra : Arrays.asList(null, new RelatednessFacet(multiStrField(2) + ":9", null))) {
      // choose single value strings, so we know both 'dv' (sweep) and 'dvhash' (no sweep) can be
      // specified choose 'id' for the parent facet, so we are guaranteed some child facets
      final TermFacet parent = new TermFacet("id", 1, 0, "skg desc", false);
      final TermFacet child = new TermFacet(soloStrField(7), 1, 0, "skg desc", false);
      parent.subFacets.put("child", child);
      if (null != extra) {
        parent.subFacets.put("skg_extra", extra);
        child.subFacets.put("skg_extra", extra);
      }
      final Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("parent", parent);

      final SolrParams facetParams =
          SolrParams.wrapDefaults(
              params("method_val", "dv", "json.facet", Facet.toJSONFacetParamValue(facets)),
              baseParams);
      // both default sweep option and explicit sweep should give same results...
      for (SolrParams sweepParams :
          Arrays.asList(
              params(),
              params("sweep_key", RelatednessAgg.SWEEP_COLLECTION, "sweep_val", "true"))) {
        final SolrParams params = SolrParams.wrapDefaults(sweepParams, facetParams);

        final NamedList<Object> parentDebug = getFacetDebug(params);
        assertEquals("id", parentDebug.get("field"));
        assertNotNull(parentDebug.get("sub-facet"));
        // may be multiples from diff shards, just use first one
        @SuppressWarnings("unchecked")
        final NamedList<Object> childDebug =
            ((List<NamedList<Object>>) parentDebug.get("sub-facet")).get(0);
        assertEquals(soloStrField(7), childDebug.get("field"));

        // these should all be true for both the parent and the child debug...
        for (NamedList<Object> debug : Arrays.asList(parentDebug, childDebug)) {
          assertEquals(FacetFieldProcessorByArrayDV.class.getSimpleName(), debug.get("processor"));
          @SuppressWarnings("unchecked")
          final NamedList<Object> sweep_debug =
              (NamedList<Object>) debug.get(SWEEP_COLLECTION_DEBUG_KEY);
          assertNotNull(sweep_debug);
          assertEquals("count", sweep_debug.get("base"));
          assertEquals(Arrays.asList("skg!fg", "skg!bg"), sweep_debug.get("accs"));
          assertEquals(Arrays.asList("skg"), sweep_debug.get("mapped"));
        }
      }
    }
  }

  /**
   * returns the <b>FIRST</b> NamedList (under the implicit 'null' FacetQuery) in the "facet-trace"
   * output of the request. Should not be used with multiple "top level" facets (the output is too
   * confusing in cloud mode to be confident where/qhy each NamedList comes from)
   */
  private NamedList<Object> getFacetDebug(final SolrParams params) {
    try {
      final QueryResponse rsp = (new QueryRequest(params)).process(getRandClient(random()));
      assertNotNull(params + " is null rsp?", rsp);
      final NamedList<?> topNamedList = rsp.getResponse();
      assertNotNull(params + " is null topNamedList?", topNamedList);

      // skip past the (implicit) top Facet query to get it's "sub-facets" (the real facets)...
      @SuppressWarnings({"unchecked"})
      final List<NamedList<Object>> facetDebug =
          (List<NamedList<Object>>) topNamedList.findRecursive("debug", "facet-trace", "sub-facet");
      assertNotNull(topNamedList + " ... null facet debug?", facetDebug);
      assertFalse(topNamedList + " ... not even one facet debug?", facetDebug.isEmpty());
      return facetDebug.get(0);
    } catch (Exception e) {
      throw new RuntimeException("query failed: " + params + ": " + e.getMessage(), e);
    }
  }

  /**
   * Test some small, handcrafted, but non-trivial queries that are easier to trace/debug then a
   * pure random monstrosity. (ie: if something obvious gets broken, this test may fail faster and
   * in a more obvious way than testRandom)
   */
  public void testBespoke() {
    { // two trivial single level facets
      Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("str", new TermFacet(multiStrField(9), UNIQUE_FIELD_VALS, 0, null, null));
      facets.put("int", new TermFacet(multiIntField(9), UNIQUE_FIELD_VALS, 0, null, null));
      assertFacetSKGsAreConsistent(
          facets, multiStrField(7) + ":11", multiStrField(5) + ":9", "*:*");
    }

    { // trivial single level facet w/sorting on skg and refinement explicitly disabled
      Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("xxx", new TermFacet(multiStrField(9), UNIQUE_FIELD_VALS, 0, "skg desc", false));
      assertFacetSKGsAreConsistent(
          facets, multiStrField(7) + ":11", multiStrField(5) + ":9", "*:*");
    }

    { // trivial single level facet w/ perSeg
      Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("xxx", new TermFacet(multiStrField(9), map("perSeg", true)));

      assertFacetSKGsAreConsistent(
          facets, multiStrField(7) + ":11", multiStrField(5) + ":9", "*:*");
    }

    { // trivial single level facet w/ prefix
      Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("xxx", new TermFacet(multiStrField(9), Map.of("prefix", "2")));

      assertFacetSKGsAreConsistent(
          facets, multiStrField(7) + ":11", multiStrField(5) + ":9", "*:*");
    }

    { // trivial single level facet w/ 2 diff ways to request "limit = (effectively) Infinite"
      // to sanity check refinement of buckets missing from other shard in both cases

      // NOTE that these two queries & facets *should* effectively identical given that the
      // very large limit value is big enough no shard will ever return that many terms,
      // but the "limit=-1" case it actually triggers slightly different code paths
      // because it causes FacetField.returnsPartial() to be "true"
      for (int limit : new int[] {999999999, -1}) {
        Map<String, TermFacet> facets = new LinkedHashMap<>();
        facets.put(
            "top_facet_limit__" + limit,
            new TermFacet(multiStrField(9), limit, 0, "skg desc", true));
        assertFacetSKGsAreConsistent(
            facets, multiStrField(7) + ":11", multiStrField(5) + ":9", "*:*");
      }
    }

    { // multi-valued facet field w/infinite limit and an extra (non-SKG / non-sweeping) stat
      final TermFacet xxx = new TermFacet(multiStrField(12), -1, 0, "count asc", false);
      xxx.subFacets.put("sum", new SumFacet(multiIntField(4)));
      final Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put("xxx", xxx);
      assertFacetSKGsAreConsistent(
          facets,
          buildORQuery(
              multiStrField(13) + ":26", multiStrField(6) + ":33", multiStrField(9) + ":24"),
          buildORQuery(
              multiStrField(4) + ":27",
              multiStrField(12) + ":18",
              multiStrField(2) + ":28",
              multiStrField(13) + ":50"),
          "*:*");
    }
  }

  public void testBespokeAllBuckets() {
    { // single level facet w/sorting on skg and allBuckets
      Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put(
          "xxx", new TermFacet(multiStrField(9), map("sort", "skg desc", "allBuckets", true)));

      assertFacetSKGsAreConsistent(
          facets, multiStrField(7) + ":11", multiStrField(5) + ":9", "*:*");
    }
  }

  public void testBespokePrefix() {
    { // trivial single level facet w/ prefix
      Map<String, TermFacet> facets = new LinkedHashMap<>();
      facets.put(
          "xxx",
          new TermFacet(
              multiStrField(9),
              map(
                  "sort", "skg desc",
                  "limit", -1,
                  "prefix", "2")));

      assertFacetSKGsAreConsistent(
          facets, multiStrField(7) + ":11", multiStrField(5) + ":9", "*:*");
    }
  }

  /**
   * Given a few explicit "structures" of requests, test many permutations of various
   * params/options. This is more complex than {@link #testBespoke} but should still be easier to
   * trace/debug then a pure random monstrosity.
   */
  public void testBespokeStructures() {
    // we don't need to test every field, just make sure we test enough fields to hit every
    // suffix...
    final int maxFacetFieldNum =
        Collections.max(
            Arrays.asList(
                MULTI_STR_FIELD_SUFFIXES.length,
                MULTI_INT_FIELD_SUFFIXES.length,
                SOLO_STR_FIELD_SUFFIXES.length,
                SOLO_INT_FIELD_SUFFIXES.length));

    for (int facetFieldNum = 0; facetFieldNum < maxFacetFieldNum; facetFieldNum++) {
      for (String facetFieldName :
          Arrays.asList(soloStrField(facetFieldNum), multiStrField(facetFieldNum))) {
        for (int limit : Arrays.asList(10, -1)) {
          for (String sort : Arrays.asList("count desc", "skg desc", "index asc")) {
            for (Boolean refine : Arrays.asList(false, true)) {
              { // 1 additional (non-SKG / non-sweeping) stat
                final TermFacet xxx =
                    new TermFacet(
                        facetFieldName,
                        map(
                            "limit", limit,
                            "overrequest", 0,
                            "sort", sort,
                            "refine", refine));
                xxx.subFacets.put("sum", new SumFacet(soloIntField(3)));
                final Map<String, TermFacet> facets = new LinkedHashMap<>();
                facets.put("xxx1", xxx);
                assertFacetSKGsAreConsistent(
                    facets,
                    buildORQuery(multiStrField(11) + ":55", multiStrField(0) + ":46"),
                    multiStrField(5) + ":9",
                    "*:*");
              }
              { // multiple SKGs
                final TermFacet xxx =
                    new TermFacet(
                        facetFieldName,
                        map(
                            "limit", limit,
                            "overrequest", 0,
                            "sort", sort,
                            "refine", refine));
                xxx.subFacets.put("skg2", new RelatednessFacet(multiStrField(2) + ":9", "*:*"));
                final Map<String, TermFacet> facets = new LinkedHashMap<>();
                facets.put("xxx2", xxx);
                assertFacetSKGsAreConsistent(
                    facets,
                    buildORQuery(multiStrField(11) + ":55", multiStrField(0) + ":46"),
                    multiStrField(5) + ":9",
                    "*:*");
              }
              { // multiple SKGs and a multiple non-SKG / non-sweeping stats
                final TermFacet xxx =
                    new TermFacet(
                        facetFieldName,
                        map(
                            "limit", limit,
                            "overrequest", 0,
                            "sort", sort,
                            "refine", refine));
                xxx.subFacets.put("minAAA", new SumFacet(soloIntField(3)));
                xxx.subFacets.put("skg2", new RelatednessFacet(multiStrField(2) + ":9", "*:*"));
                xxx.subFacets.put("minBBB", new SumFacet(soloIntField(2)));
                final Map<String, TermFacet> facets = new LinkedHashMap<>();
                facets.put("xxx3", xxx);
                assertFacetSKGsAreConsistent(
                    facets,
                    buildORQuery(multiStrField(11) + ":55", multiStrField(0) + ":46"),
                    multiStrField(5) + ":9",
                    "*:*");
              }
            }
          }
        }
      }
    }
  }

  public void testRandom() {

    final int numIters = atLeast(10);
    for (int iter = 0; iter < numIters; iter++) {
      assertFacetSKGsAreConsistent(
          TermFacet.buildRandomFacets(),
          buildRandomQuery(),
          buildRandomQuery(),
          buildRandomQuery());
    }
  }

  /**
   * Generates a random query string across the randomized fields/values in the index
   *
   * @see #randFieldValue
   * @see #field
   */
  private static String buildRandomQuery() {
    if (0 == TestUtil.nextInt(random(), 0, 10)) {
      return "*:*";
    }
    final int numClauses = TestUtil.nextInt(random(), 3, 10);
    return buildRandomORQuery(numClauses);
  }
  /** The more clauses, the more docs it's likely to match */
  private static String buildRandomORQuery(final int numClauses) {
    final String[] clauses = new String[numClauses];
    for (int c = 0; c < numClauses; c++) {
      final int fieldNum = random().nextInt(MAX_FIELD_NUM);
      // keep queries simple, just use str fields - not point of test
      clauses[c] = multiStrField(fieldNum) + ":" + randFieldValue(fieldNum);
    }
    return buildORQuery(clauses);
  }

  private static String buildORQuery(String... clauses) {
    assertTrue(0 < clauses.length);
    return "(" + String.join(" OR ", clauses) + ")";
  }

  /**
   * Given a set of term facets, and top level query strings, asserts that the results of these
   * queries are identical even when varying the <code>method_val</code> param and when varying the
   * <code>{@value RelatednessAgg#SWEEP_COLLECTION}</code> param; either by explicitly setting to
   * <code>true</code> or <code>false</code> or by changing the param key to not set it at all.
   */
  private void assertFacetSKGsAreConsistent(
      final Map<String, TermFacet> facets,
      final String query,
      final String foreQ,
      final String backQ) {
    final SolrParams basicParams =
        params(
            "rows",
            "0",
            "q",
            query,
            "fore",
            foreQ,
            "back",
            backQ,
            "json.facet",
            Facet.toJSONFacetParamValue(facets));

    log.info("Doing full run: {}", basicParams);
    try {

      // start by recording the results of the purely "default" behavior...
      final NamedList<?> expected = getFacetResponse(basicParams);

      // now loop over all permutations of processors and sweep values and compare them to the
      // "default"...
      for (FacetMethod method : EnumSet.allOf(FacetMethod.class)) {
        for (Boolean sweep : Arrays.asList(true, false, null)) {
          final ModifiableSolrParams options =
              params("method_val", method.toString().toLowerCase(Locale.ROOT));
          if (null != sweep) {
            options.add("sweep_key", RelatednessAgg.SWEEP_COLLECTION);
            options.add("sweep_val", sweep.toString());
          }

          final NamedList<?> actual =
              getFacetResponse(SolrParams.wrapAppended(options, basicParams));

          // we can't rely on a trivial assertEquals() comparison...
          //
          // the order of the sub-facet keys can change between
          // processors.  (notably: method:enum vs method:smart when sort:"index asc")
          //
          // NOTE: this doesn't ignore the order of the buckets,
          // it ignores the order of the keys in each bucket...
          final String pathToMismatch =
              BaseDistributedSearchTestCase.compare(
                  expected,
                  actual,
                  0,
                  Collections.singletonMap("buckets", BaseDistributedSearchTestCase.UNORDERED));
          if (null != pathToMismatch) {
            log.error("{}: expected = {}", options, expected);
            log.error("{}: actual = {}", options, actual);
            fail("Mismatch: " + pathToMismatch + " using " + options);
          }
        }
      }
    } catch (AssertionError e) {
      throw new AssertionError(basicParams + " ===> " + e.getMessage(), e);
    } finally {
      log.info("Ending full run");
    }
  }

  /**
   * We ignore {@link QueryResponse#getJsonFacetingResponse()} because it isn't as useful for doing
   * a "deep equals" comparison across requests
   */
  private NamedList<?> getFacetResponse(final SolrParams params) {
    try {
      final QueryResponse rsp = (new QueryRequest(params)).process(getRandClient(random()));
      assertNotNull(params + " is null rsp?", rsp);
      final NamedList<Object> topNamedList = rsp.getResponse();
      assertNotNull(params + " is null topNamedList?", topNamedList);
      final NamedList<?> facetResponse = (NamedList<?>) topNamedList.get("facets");
      assertNotNull("null facet results?", facetResponse);
      assertEquals(
          "numFound mismatch with top count?",
          rsp.getResults().getNumFound(),
          ((Number) facetResponse.get("count")).longValue());

      return facetResponse;

    } catch (Exception e) {
      throw new RuntimeException("query failed: " + params + ": " + e.getMessage(), e);
    }
  }

  private static interface Facet { // Mainly just a Marker Interface

    /**
     * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param
     * value to use for testing them against in a solr request.
     */
    public static String toJSONFacetParamValue(final Map<String, ? extends Facet> facets) {
      assertNotNull(facets);
      assertFalse(facets.isEmpty());

      return JSONUtil.toJSON(facets, -1); // no newlines
    }
  }

  /**
   * trivial facet that is not SKG (and doesn't have any of its special behavior) for the purposes
   * of testing how TermFacet behaves with a mix of sub-facets.
   */
  private static final class SumFacet implements Facet {
    private final String field;

    public SumFacet(final String field) {
      this.field = field;
    }

    @Override
    public String toString() { // used in JSON by default
      return "sum(" + field + ")";
    }

    public static SumFacet buildRandom() {
      final int fieldNum = random().nextInt(MAX_FIELD_NUM);
      final boolean multi = random().nextBoolean();
      return new SumFacet(multi ? multiIntField(fieldNum) : soloIntField(fieldNum));
    }
  }

  /**
   * Trivial data structure for modeling a simple <code>relatedness()</code> facet that can be
   * written out as a json.facet param.
   *
   * <p>Doesn't do any string escaping or quoting, so don't use whitespace or reserved json
   * characters
   *
   * <p>The specified fore/back queries will be wrapped in localparam syntax in the resulting json,
   * unless they are 'null' in which case <code>$fore</code> and <code>$back</code> refs will be
   * used in their place, and must be set as request params (this allows "random" facets to still
   * easily trigger the nested facets re-using the same fore/back set for SKG situation)
   *
   * <p>The JSON for all of these facets includes a <code>${sweep_key:xxx}</code> (which will be
   * ignored by default) and <code>${sweep_val:yyy}</code> which may be set as params on each
   * request to override the implicit default sweeping behavior of the underlying SKGAcc.
   */
  private static final class RelatednessFacet implements Facet, Writable {
    public final Map<String, Object> jsonData = new LinkedHashMap<>();

    /** Assumes null for fore/back queries w/no options */
    public RelatednessFacet() {
      this(null, null, Collections.emptyMap());
    }
    /** Assumes no options */
    public RelatednessFacet(final String foreQ, final String backQ) {
      this(foreQ, backQ, Collections.emptyMap());
    }

    public RelatednessFacet(
        final String foreQ, final String backQ, final Map<String, Object> options) {
      assertNotNull(options);

      final String f = null == foreQ ? "$fore" : "{!v='" + foreQ + "'}";
      final String b = null == backQ ? "$back" : "{!v='" + backQ + "'}";

      jsonData.putAll(options);

      // we don't allow these to be overridden by options, so set them now...
      jsonData.put("type", "func");
      jsonData.put("func", "relatedness(" + f + "," + b + ")");
      jsonData.put("${sweep_key:xxx}", "${sweep_val:yyy}");
    }

    @Override
    public void write(JSONWriter writer) {
      writer.write(jsonData);
    }

    public static RelatednessFacet buildRandom() {

      final Map<String, Object> options = new LinkedHashMap<>();
      if (random().nextBoolean()) {
        options.put("min_popularity", "0.001");
      }

      // bias this in favor of null fore/back since that's most realistic for typical nested facets
      final boolean simple = random().nextBoolean();
      final String fore = simple ? null : buildRandomORQuery(TestUtil.nextInt(random(), 1, 5));
      final String back = simple ? null : buildRandomORQuery(TestUtil.nextInt(random(), 1, 9));

      return new RelatednessFacet(fore, back, options);
    }
  }

  /**
   * Trivial data structure for modeling a simple terms facet that can be written out as a
   * json.facet param. Since the point of this test is SKG, every TermFacet implicitly has one fixed
   * "skg" subFacet, but that can be overridden by the caller
   *
   * <p>Doesn't do any string escaping or quoting, so don't use whitespace or reserved json
   * characters
   *
   * <p>The resulting facets all specify a <code>method</code> of <code>${method_val:smart}</code>
   * which may be overridden via request params.
   */
  private static final class TermFacet implements Facet, Writable {

    public final Map<String, Object> jsonData = new LinkedHashMap<>();
    public final Map<String, Facet> subFacets = new LinkedHashMap<>();

    /**
     * @param field must be non-null
     * @param options can set any of options used in a term facet other than field or (sub) facets
     */
    public TermFacet(final String field, final Map<String, Object> options) {
      assertNotNull(field);

      jsonData.put("method", "${method_val:smart}");

      jsonData.putAll(options);

      // we don't allow these to be overridden by options, so set them now...
      jsonData.put("type", "terms");
      jsonData.put("field", field);
      jsonData.put("facet", subFacets);

      subFacets.put("skg", new RelatednessFacet());
    }

    /** all params except field can be null */
    public TermFacet(
        String field, Integer limit, Integer overrequest, String sort, Boolean refine) {
      this(field, map("limit", limit, "overrequest", overrequest, "sort", sort, "refine", refine));
    }

    @Override
    public void write(JSONWriter writer) {
      writer.write(jsonData);
    }

    /**
     * Generates a random TermFacet that does not contain any random sub-facets beyond a single
     * consistent "skg" stat
     */
    public static TermFacet buildRandom() {
      final String sort = randomSortParam(random());
      final String facetField = randomFacetField(random());
      return new TermFacet(
          facetField,
          map(
              "limit", randomLimitParam(random()),
              "overrequest", randomOverrequestParam(random()),
              "prefix", randomPrefixParam(random(), facetField),
              "perSeg", randomPerSegParam(random()),
              "sort", sort,
              "prelim_sort", randomPrelimSortParam(sort),
              "allBuckets", randomAllBucketsParam(random()),
              "refine", randomRefineParam(random())));
    }

    /**
     * Factory method for generating some random facets.
     *
     * <p>For simplicity, each facet will have a unique key name.
     */
    public static Map<String, TermFacet> buildRandomFacets() {
      // for simplicity, use a unique facet key regardless of depth - simplifies verification
      // and let's us enforce a hard limit on the total number of facets in a request
      AtomicInteger keyCounter = new AtomicInteger(0);

      final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
      return buildRandomFacets(keyCounter, maxDepth);
    }

    /**
     * picks a random field to facet on.
     *
     * @see #field
     * @return field name, never null
     */
    public static String randomFacetField(final Random r) {
      final int fieldNum = r.nextInt(MAX_FIELD_NUM);
      switch (r.nextInt(4)) {
        case 0:
          return multiStrField(fieldNum);
        case 1:
          return multiIntField(fieldNum);
        case 2:
          return soloStrField(fieldNum);
        case 3:
          return soloIntField(fieldNum);
        default:
          throw new RuntimeException("Broken case statement");
      }
    }

    /**
     * picks a random value for the "allBuckets" param, biased in favor of interesting test cases
     * This bucket should be ignored by relatedness, but inclusion should not cause any problems (or
     * change the results)
     *
     * <p><b>NOTE:</b> allBuckets is meaningless in conjunction with the <code>STREAM</code>
     * processor, so this method always returns null if sort is <code>index asc</code>.
     *
     * @return a Boolean, may be null
     */
    public static Boolean randomAllBucketsParam(final Random r) {
      switch (r.nextInt(4)) {
        case 0:
          return true;
        case 1:
          return false;
        case 2:
        case 3:
          return null;
        default:
          throw new RuntimeException("Broken case statement");
      }
    }

    /**
     * picks a random value for the "refine" param, biased in favor of interesting test cases
     *
     * @return a Boolean, may be null
     */
    public static Boolean randomRefineParam(final Random r) {

      switch (r.nextInt(3)) {
        case 0:
          return null;
        case 1:
          return true;
        case 2:
          return false;
        default:
          throw new RuntimeException("Broken case statement");
      }
    }

    /**
     * picks a random value for the "perSeg" param, biased in favor of interesting test cases
     *
     * @return a Boolean, may be null
     */
    public static Boolean randomPerSegParam(final Random r) {

      switch (r.nextInt(4)) {
        case 0:
          return true;
        case 1:
          return false;
        case 2:
        case 3:
          return null;
        default:
          throw new RuntimeException("Broken case statement");
      }
    }

    /**
     * picks a random value for the "prefix" param, biased in favor of interesting test cases
     *
     * @return a valid prefix value, may be null
     */
    public static String randomPrefixParam(final Random r, final String facetField) {

      if (facetField.contains("multi_i") || facetField.contains("solo_i")) {
        // never used a prefix on a numeric field
        return null;
      }
      assertTrue(
          "possible facet fields have changed, breaking test",
          facetField.contains("multi_s") || facetField.contains("solo_s"));

      switch (r.nextInt(5)) {
        case 0:
          return "2";
        case 1:
          return "3";
        case 2:
        case 3:
        case 4:
          return null;
        default:
          throw new RuntimeException("Broken case statement");
      }
    }

    /**
     * picks a random value for the "sort" param, biased in favor of interesting test cases. Assumes
     * every TermFacet will have at least one "skg" stat
     *
     * @return a sort string (w/direction), or null to specify nothing (trigger default behavior)
     * @see #randomAllBucketsParam
     * @see #randomPrelimSortParam
     */
    public static String randomSortParam(final Random r) {

      final String dir = random().nextBoolean() ? "asc" : "desc";
      switch (r.nextInt(4)) {
        case 0:
          return null;
        case 1:
          return "count " + dir;
        case 2:
          return "skg " + dir;
        case 3:
          return "index " + dir;
        default:
          throw new RuntimeException("Broken case statement");
      }
    }
    /**
     * picks a random value for the "prelim_sort" param, biased in favor of interesting test cases.
     *
     * @return a sort string (w/direction), or null to specify nothing (trigger default behavior)
     */
    public static String randomPrelimSortParam(final String sort) {

      if (null != sort && sort.startsWith("skg") && 1 == TestUtil.nextInt(random(), 0, 3)) {
        return "count desc";
      }
      return null;
    }
    /**
     * picks a random value for the "limit" param, biased in favor of interesting test cases
     *
     * @return a number to specify in the request, or null to specify nothing (trigger default
     *     behavior)
     * @see #UNIQUE_FIELD_VALS
     */
    public static Integer randomLimitParam(final Random r) {

      final int limit = 1 + r.nextInt((int) (UNIQUE_FIELD_VALS * 1.5F));

      if (1 == TestUtil.nextInt(random(), 0, 3)) {
        // bias in favor of just using default
        return null;
      }

      if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
        return -1; // unlimited
      }

      return limit;
    }

    /**
     * picks a random value for the "overrequest" param, biased in favor of interesting test cases.
     *
     * @return a number to specify in the request, or null to specify nothing (trigger default
     *     behavior)
     * @see #UNIQUE_FIELD_VALS
     */
    public static Integer randomOverrequestParam(final Random r) {

      switch (r.nextInt(10)) {
        case 0:
        case 1:
        case 2:
        case 3:
          return 0; // 40% of the time, disable overrequest to better stress refinement
        case 4:
        case 5:
          return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
        case 6:
          return r.nextInt(
              Integer.MAX_VALUE); // 10%: completely random value, statisticaly more than enough
        default:
          break;
      }
      // else.... either leave param unspecified (or redundantly specify the -1 default)
      return r.nextBoolean() ? null : -1;
    }

    /**
     * recursive helper method for building random facets
     *
     * @param keyCounter used to ensure every generated facet has a unique key name
     * @param maxDepth max possible depth allowed for the recursion, a lower value may be used
     *     depending on how many facets are returned at the current level.
     */
    private static Map<String, TermFacet> buildRandomFacets(
        AtomicInteger keyCounter, int maxDepth) {
      final int numFacets =
          Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
      Map<String, TermFacet> results = new LinkedHashMap<>();
      for (int i = 0; i < numFacets; i++) {
        // a hard limit on the total number of facets (regardless of depth) to reduce OOM risk
        if (keyCounter.get() < 3) {

          final TermFacet facet = TermFacet.buildRandom();

          results.put("facet_" + keyCounter.incrementAndGet(), facet);
          if (0 < maxDepth) {
            // if we're going wide, don't go deep
            final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
            facet.subFacets.putAll(
                buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
          }

          // we get one implicit RelatednessFacet automatically,
          // randomly add 1 or 2 more ... 3/5th chance of being '0'
          final int numExtraSKGStats = Math.max(0, TestUtil.nextInt(random(), -2, 2));
          for (int skgId = 0; skgId < numExtraSKGStats; skgId++) {
            // sometimes we overwrite the trivial default "skg" with this one...
            final String key =
                (0 == skgId && 0 == TestUtil.nextInt(random(), 0, 5)) ? "skg" : "skg" + skgId;
            facet.subFacets.put(key, RelatednessFacet.buildRandom());
          }

          if (1 == TestUtil.nextInt(random(), 0, 4)) {
            // occasionally add in a non-SKG related stat...
            facet.subFacets.put("sum", SumFacet.buildRandom());
          }
        }
      }
      return results;
    }
  }

  /**
   * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed at a node
   * in our cluster
   */
  public static SolrClient getRandClient(Random rand) {
    int numClients = CLIENTS.size();
    int idx = TestUtil.nextInt(rand, 0, numClients);

    return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
  }

  /**
   * Uses a random SolrClient to execute a request and returns only the numFound
   *
   * @see #getRandClient
   */
  public static long getNumFound(final SolrParams req) throws SolrServerException, IOException {
    return getRandClient(random()).query(req).getResults().getNumFound();
  }

  public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
    assertNotNull(client.getDefaultCollection());
    AbstractDistribZkTestBase.waitForRecoveriesToFinish(
        client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330);
  }

  /** helper macro: fails on null keys, skips pairs with null values */
  public static Map<String, Object> map(Object... pairs) {
    if (0 != pairs.length % 2) throw new IllegalArgumentException("uneven number of arguments");
    final Map<String, Object> map = new LinkedHashMap<>();
    for (int i = 0; i < pairs.length; i += 2) {
      final Object key = pairs[i];
      final Object val = pairs[i + 1];
      if (null == key) throw new NullPointerException("argument " + i);
      if (null == val) continue;

      map.put(key.toString(), val);
    }
    return map;
  }
}
