/* See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * Esri Inc. licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.esri.gpt.catalog.arcgis.metadata;
import com.esri.gpt.catalog.publication.ProcessedRecord;
import com.esri.gpt.catalog.publication.ProcessingContext;
import com.esri.gpt.catalog.publication.PublicationRecord;
import com.esri.gpt.catalog.publication.ResourceProcessor;

import com.esri.arcgisws.ServiceCatalogBindingStub;
import com.esri.arcgisws.ServiceDescription;
import com.esri.arcgisws.runtime.exception.ArcGISWebServiceException;
import com.esri.gpt.control.webharvest.AccessException;
import com.esri.gpt.control.webharvest.IterationContext;
import com.esri.gpt.control.webharvest.common.CommonResult;
import com.esri.gpt.framework.http.crawl.CrawlLocker;
import com.esri.gpt.framework.resource.adapters.FlatResourcesAdapter;
import com.esri.gpt.framework.resource.adapters.LimitedLengthResourcesAdapter;
import com.esri.gpt.framework.resource.adapters.PublishablesAdapter;
import com.esri.gpt.framework.resource.api.Native;
import com.esri.gpt.framework.resource.api.Publishable;
import com.esri.gpt.framework.resource.api.Resource;
import com.esri.gpt.framework.resource.query.Criteria;
import com.esri.gpt.framework.resource.query.Query;
import com.esri.gpt.framework.resource.query.Result;
import com.esri.gpt.framework.robots.Bots;
import static com.esri.gpt.framework.robots.BotsUtils.parser;
import static com.esri.gpt.framework.robots.BotsUtils.transformUrl;
import com.esri.gpt.framework.security.credentials.UsernamePasswordCredentials;
import com.esri.gpt.framework.util.ReadOnlyIterator;
import com.esri.gpt.framework.util.Val;

import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Processes resources associated with an ArcGIS server.
 */
public class AGSProcessor extends ResourceProcessor {
  
  /** class variables ========================================================= */
  
  /** Logger */
  private static final Logger LOGGER = Logger.getLogger(AGSProcessor.class.getName());
  
  /** instance variables ====================================================== */
  private ServiceHandlerFactory factory = new ServiceHandlerFactory();
  private AGSTarget             target = new AGSTarget();
  private UsernamePasswordCredentials credentials;
  private Long throttleDelay;

  /** constructors ============================================================ */

  /**
   * Constructs with an associated processing context.
   * @param context the procesing context
   */
  public AGSProcessor(ProcessingContext context) {
    super(context);
    if (context.getTemplate() == null) {
      PublicationRecord template = new PublicationRecord();
      template.setUpdateOnlyIfXmlHasChanged(true);
      context.setTemplate(template);
    }
  }
  
  /** properties ============================================================== */
  
  /**
   * Gets the ArcGIS server service handler factory.
   * @return the factory
   */
  public ServiceHandlerFactory getHandlerFactory() {
    return this.factory;
  }
  
  /**
   * Gets the targeted ArcGIS server and resource.
   * @return the target
   */
  public AGSTarget getTarget() {
    return this.target;
  }

  /**
   * Gets credentials.
   * @return credentials
   */
  public UsernamePasswordCredentials getCredentials() {
    return credentials;
  }

  /**
   * Sets credentials.
   * @param credentials credentials
   */
  public void setCredentials(UsernamePasswordCredentials credentials) {
    this.credentials = credentials;
  }

  /**
   * Gets throttle delay.
   * <p>
   * If set it will override "Crawl-Delay" from robots.txt if robots.txt present.
   * @return throttle delay in milliseconds or {@code null} if no throttling required
   */
  public Long getThrottleDelay() {
    return throttleDelay;
  }

  /**
   * Sets throttle delay.
   * <p>
   * If set it will override "Crawl-Delay" from robots.txt if robots.txt present.
   * @param throttleDelay throttle delay in milliseconds or {@code null} if no throttling required
   */
  public void setThrottleDelay(Long throttleDelay) {
    this.throttleDelay = throttleDelay;
  }
  
  /** methods ================================================================= */
  
  /**
   * Interrogates the character response from a target resource URL attempting to
   * determine the REST and SOAP endpoints for an ArcGIS server services catalog.
   * @param url the target URL associated with the resource being interrogated
   * @param response the character based response previously returned from the target URL
   * @return <code>true</code> if the target was recognized as an ArcGIS server endpoint
   */
  public boolean interrogate(URL url, String response) throws IOException {
    AGSInterrogator interragator = new AGSInterrogator(this.getContext(),this.getTarget());
    interragator.interrogate(url,response);
    getTarget().updateTargetSoapUrl();
    return getTarget().getWasRecognized();
  }
  
  /**
   * Invokes processing against the resource.
   * @throws Exception if an exception occurs
   */
  @Override
  public void process() throws Exception {
    
   /*
    * ServiceCatalogBindingStub
    *   name:          dc:title
    *   description:   dc:description
    *   resource.url:  dct:references 
    *                  scheme=urn:x-esri:specification:ServiceType:ArcGIS:type
    *                  value is typically the rest endpoint for the service
    *   type:          typically dc:subject -> keyword Lucene field is dataTheme
    *   parentType:    not indexed
    *   capabilities:  not indexed
    *   sourceUri:     service rest url (by default)
    *   
    * GeocodeServerBindingStub
    *   keywords: GeocodeServer,geographicService,service,locator,geocode,geocoder
    *   
    * GeoDataServerBindingStub
    *   keywords: GeoDataServer,geographicService,service
    *   per dataElement: publish element.getMetadata
    *                    sourceUri=serviceRestUrl/element.getName
    *   
    * GeometryServerBindingStub
    *   keywords: GeometryServer,geographicService,service,geometry,projection
    *   
    * GlobeServerBindingStub
    *   keywords: GlobeServer,liveData,service,globe
    *   per layer: dct:abstract/rdf:value@rdf:resource=service.layername
    *   
    * GPServerBindingStub
    *   keywords: GPServer,geographicService,service,geoprocessing
    *   per task: add task name as keyword, set service envelope is applicable
    *   
    * ImageServerBindingStub
    *   keywords: ImageServer,liveData,service,image
    *   imageServiceInfo.description: dc:description
    *   imageServiceInfo.extent: ows:WGS84BoundingBox
    * 
    * MapServerBindingStub
    *   keywords: ImageServer,liveData,service,image
    *   mapServerInfo.description: dc:description
    *   mapServerInfo.fullExtent: ows:WGS84BoundingBox
    *   thumbnail.url: serviceRestUrl/export?size=256,256&f=image
    *   documentInfo['Title']: dc:title
    *   documentInfo['Author']: dc:creator
    *   documentInfo['Comments']: dct:abstract/rdf:value@rdf:resource=mxd.comments
    *   documentInfo['Subject']: dct:abstract/rdf:value@rdf:resource=mxd.subject
    *   documentInfo['Category']: dct:abstract/rdf:value@rdf:resource=mxd.category
    *   documentInfo['Keywords']: dc:subject
    *   per layer: dct:abstract/rdf:value@rdf:resource=service.layername
    * 
    * MobileServerBindingStub
    *   keywords: MobileServer,liveData,service
    *   
    * NAServerBindingStub
    *   keywords: NAServer,geographicService,service,network,route
    *   
    * WCSServer
    *   keywords: WCSServer,liveData,service
    *   resource.url: soapEndpoint?request=GetCapabilities&service=WCSServer
    *   plus parent service metadata
    *   
    * WFSServer
    *   keywords: WFSServer,liveData,service
    *   resource.url: soapEndpoint?request=GetCapabilities&service=WFSServer
    *   plus parent service metadata
    *   
    * WMSServer
    *   keywords: WMSServer,liveData,service
    *   resource.url: soapEndpoint?request=GetCapabilities&service=WMSServer
    *   plus parent service metadata
    *   
    */
    
    String restUrl = this.getTarget().getRestUrl();
    String soapUrl = this.getTarget().getSoapUrl();
    AGSTarget.TargetType targetType = this.getTarget().getTargetType();
    
    if ((targetType != null) && targetType.equals(AGSTarget.TargetType.ROOT)) {
      this.collectExistingSourceURIs(restUrl,soapUrl);
    }
    
    // TODO: check the TargetType
    // determine the target (entire server, a folder or a service)
    getTarget().updateTargetSoapUrl();
    String targetSoapUrl = getTarget().getTargetSoapUrl();
    boolean matchAll = targetSoapUrl.equals(soapUrl);
    boolean checkFolder = !targetSoapUrl.endsWith("Server");
    
    // loop through the service descriptions
    ServiceCatalogBindingStub stub = new ServiceCatalogBindingStub(soapUrl);    
    for (ServiceDescription desc: stub.getServiceDescriptions()) {
      if (Thread.currentThread().isInterrupted()) return;
      String currentSoapUrl = desc.getUrl();
      String currentRestUrl = currentSoapUrl.replace(soapUrl,restUrl);
 
      // determine if there is a metch
      boolean matchesTarget = false;
      if (!matchAll) {
        matchesTarget = targetSoapUrl.equals(currentSoapUrl);
        if (!matchesTarget && checkFolder) {
          matchesTarget = currentSoapUrl.startsWith(targetSoapUrl+"/");
        }
      }
      if (matchAll || matchesTarget) {
        
        // make a handler for this service type
        ServiceHandler handler = this.factory.makeHandler(desc.getType());
        if (handler != null) {
          handler.setCredentials(getCredentials());
      
          // initialize service information
          ServiceInfo info = handler.createServiceInfo(null, desc, currentRestUrl, currentSoapUrl);
          
          // collect
          try {
            LOGGER.log(Level.FINER, "Collecting metadata for: {0}", Val.stripControls(info.getSoapUrl()));
            handler.collectMetadata(this,info);
          } catch (Exception e) {
            ProcessedRecord processedRcord = new ProcessedRecord();
            processedRcord.setSourceUri(info.getResourceUrl());
            processedRcord.setStatusType(ProcessedRecord.StatusType.FAILED);
            processedRcord.setException(e,this.getContext().getMessageBroker());
            this.getContext().incrementNumberFailed();
            this.getContext().setLastException(e);
            this.getContext().getProcessedRecords().add(processedRcord);
            LOGGER.log(Level.FINER,"Error\n"+Val.stripControls(processedRcord.getSourceUri()),e);
          }
          
          // publish
          try {
            LOGGER.log(Level.FINER, "Publishing metadata for: {0}", info.getResourceUrl());
            handler.publishMetadata(this,info);
          } catch (Exception e) {
            LOGGER.log(Level.FINER,"Error during publication.",e);
          }
          
          // break if we have found a single targetted service
          if (!matchAll && !checkFolder) {
            break;
          }
          
        }
      }
    }
    
    // cleanup unreferenced source URIs
    if ((targetType != null) && targetType.equals(AGSTarget.TargetType.ROOT)) {
      this.deleteUnreferencedSourceURIs();
    }
    
  }

  @Override
  public Query createQuery(final IterationContext context, final Criteria criteria) {
    return new Query() {
      @Override
      public Result execute() {
        ResourceFolders folders = createResourceFolders(context);
        return new CommonResult(new LimitedLengthResourcesAdapter(folders, criteria.getMaxRecords()));
      }
    };
  }

  @Override
  public Native getNativeResource(IterationContext context) {
    ResourceFolders folders = createResourceFolders(context);
    for (Publishable publishable : new PublishablesAdapter(new FlatResourcesAdapter(folders))) {
      if (publishable instanceof Native) {
        return (Native)publishable;
      }
      break;
    }
    return null;
  }

  /**
   * Resolves throttle delay.
   * @param bots robots.txt or {@code null} if robots.txt unavailable
   * @return throttle delay in milliseconds or {@code null} if no throttling required
   */
  protected Long resolveThrottleDelay(Bots bots) {
    return getThrottleDelay()!=null? 
            getThrottleDelay(): 
            bots!=null && bots.getCrawlDelay()!=null?  
              1000L*bots.getCrawlDelay(): 
              null;
  }
  
  /**
   * Normalizes URL by removing 'wsdl'.
   * @param url URL
   * @return normalized URL
   */
  private String normalizeUrl(String url) {
    Pattern wsdlPattern = Pattern.compile("\\?wsdl$", Pattern.CASE_INSENSITIVE);
    Matcher wsdlMatcher = wsdlPattern.matcher(Val.chkStr(url));
    String wsdlResult = wsdlMatcher.replaceFirst("");
    
    Pattern servicesPattern = Pattern.compile("services\\?wsdl/", Pattern.CASE_INSENSITIVE);
    Matcher servicesMatcher = servicesPattern.matcher(wsdlResult);
    String servicesResult = servicesMatcher.replaceAll("");
    
    return servicesResult.replaceAll("/+$", "");
    
    /*
    return Pattern.compile("services\\?wsdl/", Pattern.CASE_INSENSITIVE).matcher(
      Pattern.compile("\\?wsdl$", Pattern.CASE_INSENSITIVE).matcher(
        Val.chkStr(url)
      ).replaceFirst("")
    ).replaceFirst("");
    */
  }

  /**
   * Extracts root URL.
   * @param url URL
   * @return root URL
   */
  private String extractRootUrl(String url) {
    url = Val.chkStr(url);
    try {
      URI uri = new URI(url);
      Matcher matcher = Pattern.compile("^/[^/]*(/services)?",Pattern.CASE_INSENSITIVE).matcher(uri.getPath());
      if (matcher.find()) {
        return uri.getScheme() + "://" + uri.getAuthority() + matcher.group();
      } else {
        return url;
      }
    } catch (Exception ex) {
      return url;
    }
  }

  /**
   * Reads service descriptions.
   * @param context iteration context
   * @return array of service descriptions
   * @throws ArcGISWebServiceException if accessing service descriptions 
   */
  private ServiceDescription[] readServiceDescriptions(IterationContext context) throws ArcGISWebServiceException {
    String soapUrl = extractRootUrl(getTarget().getSoapUrl());
    Bots bots = null;
    if (context!=null) {
      bots = context.getRobotsTxt();
      soapUrl = transformUrl(bots,soapUrl);
      try {
        context.assertAccess(soapUrl);
      } catch (AccessException ex) {
        throw new ArcGISWebServiceException(String.format("Access to: %s forbidden by robots.txt", getTarget().getSoapUrl()), ex);
      }
      if (bots!=null) {
        CrawlLocker.getInstance().enterServer(soapUrl, resolveThrottleDelay(bots));
      }
    }
    ServiceCatalogBindingStub stub = new ServiceCatalogBindingStub(soapUrl);
    if (bots!=null && !parser().getUserAgent().isEmpty()) {
      HashMap<String,List<String>> headers = new HashMap<String, List<String>>();
      headers.put("User-Agent", Arrays.asList(new String[]{parser().getUserAgent()}));
      stub.addHTTPRequestHeaders(headers);
    }
    ServiceDescription[] descriptors = stub.getServiceDescriptions();
    return descriptors;
  }
  
  /**
   * Creates resource folders.
   * @param context iteration context
   * @return resource folders
   */
  private ResourceFolders createResourceFolders(IterationContext context) {
    try {
      ServiceDescription[] descriptors = readServiceDescriptions(context);
      return new ResourceFolders(context, factory, descriptors);
    } catch (ArcGISWebServiceException ex) {
      context.onIterationException(ex);
      return new ResourceFolders(context, factory, new ServiceDescription[]{});
    }
  }

  /**
   * ArcGIS folders.
   */
  private class ResourceFolders implements Iterable<IServiceInfoProvider> {
  /** iteration context */
  private IterationContext context;
  /** service handler factory */
  private ServiceHandlerFactory factory;
  /** service descriptors */
  private ServiceDescription[] descriptors;
  /** normalized target SOAP URL */
  private String normalizedTargetSoapUrl;
  /** indicator to match everything or only selected service */
  private boolean matchAll;
  /** indicator to check folder */
  private boolean checkFolder;
  
  private HashMap<ServiceDescription,ServiceDescription> childToParent = new HashMap<ServiceDescription, ServiceDescription>();
  private HashMap<ServiceDescription,ServiceInfo> sdToSi = new HashMap<ServiceDescription, ServiceInfo>();

  /**
   * Creates instance of the folders.
   * @param context iteration context
   * @param factory service handler factory
   * @param descriptors service descriptors
   */
  public ResourceFolders(IterationContext context, ServiceHandlerFactory factory, ServiceDescription[] descriptors) {
    if (context==null) throw new IllegalArgumentException("No context provided.");
    if (factory==null) throw new IllegalArgumentException("No factory provided.");
    if (descriptors==null) throw new IllegalArgumentException("No descriptors provided.");
    this.context = context;
    this.factory = factory;
    this.descriptors = descriptors;
    this.normalizedTargetSoapUrl = normalizeUrl(getTarget().getTargetSoapUrl());
    this.matchAll = normalizedTargetSoapUrl.equalsIgnoreCase(extractRootUrl(getTarget().getSoapUrl()));
    this.checkFolder = !normalizedTargetSoapUrl.endsWith("Server");
    
    HashMap<String,ServiceDescription> urlToSD = new HashMap<String, ServiceDescription>();
    for (ServiceDescription sd: descriptors) {
      String url = sd.getUrl();
      urlToSD.put(url, sd);
    }
    
    for (ServiceDescription sd: descriptors) {
      if (sd.getParentType().isEmpty()) continue;
      int index = sd.getUrl().indexOf(sd.getParentType()) + sd.getParentType().length();
      String url = sd.getUrl().substring(0, index);
      
      ServiceDescription parentSD = urlToSD.get(url);
      childToParent.put(sd, parentSD);
    }
  }

  public Iterator<IServiceInfoProvider> iterator() {
    return new AGSRecordsIterator();
  }

  /**
   * ArcGIS folders iterator.
   */
  private class AGSRecordsIterator extends ReadOnlyIterator<IServiceInfoProvider> {
  /** index of the current folder */
  private int index = -1;
  /** service handler */
  private ServiceHandler handler = null;
  /** service info */
  private ServiceInfo info = null;

  /**
   * Resets current service information.
   */
  private void reset() {
    handler = null;
    info = null;
  }

  public boolean hasNext() {
    if (handler!=null && info!=null) return true;
    if (index+1>=descriptors.length) return false;
    ServiceDescription desc = descriptors[++index];
    String currentSoapUrl = desc.getUrl();
    String currentRestUrl = Pattern.compile("\\Q"+getTarget().getSoapUrl()+"\\E", Pattern.CASE_INSENSITIVE).matcher(currentSoapUrl).replaceFirst(getTarget().getRestUrl());
    boolean matchesTarget = false;
    if (!matchAll) {
      matchesTarget = normalizedTargetSoapUrl.equalsIgnoreCase(currentSoapUrl);
      if (!matchesTarget && checkFolder) {
        matchesTarget = currentSoapUrl.toLowerCase().startsWith(normalizedTargetSoapUrl.toLowerCase()+"/");
      }
    }
    if (!(matchAll || matchesTarget)) return hasNext();
    handler = factory.makeHandler(desc.getType());
    if (handler==null) return hasNext();
    handler.setCredentials(getCredentials());
    
    // get parent description if available for the current description
    ServiceDescription parentDesc = childToParent.get(desc);
    // get service info for the parent
    ServiceInfo parentInfo = sdToSi.get(parentDesc);
    
    // create servcice info for the current service description
    info = handler.createServiceInfo(parentInfo, desc, currentRestUrl, currentSoapUrl);
    
    // store mapping between service descritpion and service info
    sdToSi.put(desc, info);

    return true;
  }

  @Override
  public IServiceInfoProvider next() {
    final ResourceRecordsFamily family = new ResourceRecordsFamily(context, factory, handler, info, !matchAll && !checkFolder);
    reset();
    return new ServiceInfoProvider(info) {
      @Override
      public Iterable<Resource> getNodes() {
        return family;
      }
    };
  }
  }
  }

  /**
   * Family of the records. This is a collection of records derived from the same
   * service URL.
   */
  private class ResourceRecordsFamily implements Iterable<Resource> {
  /** iteration context */
  private IterationContext context;
  /** service handler factory */
  private ServiceHandlerFactory factory;
  /** service handler */
  private ServiceHandler handler;
  /** service info */
  private ServiceInfo info;
  /** is native */
  private boolean isNative;

  /**
   * Creates instance of the records family.
   * @param context iteration context
   * @param factory service handler factory
   * @param handler service handler
   * @param info service info
   * @param isNative <code>true</code> to append native record
   */
  public ResourceRecordsFamily(IterationContext context, ServiceHandlerFactory factory, ServiceHandler handler, ServiceInfo info, boolean isNative) {
    if (context==null) throw new IllegalArgumentException("No context provided.");
    if (factory==null) throw new IllegalArgumentException("No factory provided.");
    if (handler==null) throw new IllegalArgumentException("No handler provided.");
    if (info==null) throw new IllegalArgumentException("No info provided.");
    this.context = context;
    this.factory = factory;
    this.handler = handler;
    this.info = info;
    this.isNative = isNative;
  }

  public Iterator<Resource> iterator() {
    ArrayList<Resource> recs = new ArrayList<Resource>();
    try {
      handler.appendRecord(recs, factory, info, isNative);
    } catch (Exception ex) {
      context.onIterationException(ex);
    }
    return recs.iterator();
  }
  }

}
