/*
	$Id: GalFlatFileImporter.java 2324 2006-05-24 09:50:59Z nicklas $

	Copyright (C) Authors contributing to this file.
	
	This file is part of BASE - BioArray Software Environment.
	Available at http://base.thep.lu.se/

	BASE is free software; you can redistribute it and/or
	modify it under the terms of the GNU General Public License
	as published by the Free Software Foundation; either version 2
	of the License, or (at your option) any later version.

	BASE is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with this program; if not, write to the Free Software
	Foundation, Inc., 59 Temple Place - Suite 330,
	Boston, MA  02111-1307, USA.
*/


package org.fhcrc.genomics.baseExtensions;

import net.sf.basedb.core.ArrayDesign;
import net.sf.basedb.core.ArrayDesignBlock;
import net.sf.basedb.core.BaseException;
import net.sf.basedb.core.BlockInfo;
import net.sf.basedb.core.DbControl;
import net.sf.basedb.core.FeatureBatcher;
import net.sf.basedb.core.FileType;
import net.sf.basedb.core.ItemContext;
import net.sf.basedb.core.ReporterBatcher;
import net.sf.basedb.core.InvalidDataException;
import net.sf.basedb.core.ItemParameterType;
import net.sf.basedb.core.Job;
import net.sf.basedb.core.PermissionDeniedException;
import net.sf.basedb.core.PluginParameter;
import net.sf.basedb.core.RequestInformation;
import net.sf.basedb.core.StringParameterType;
import net.sf.basedb.core.Item;
import net.sf.basedb.core.data.FeatureData;
import net.sf.basedb.core.data.ReporterData;
import net.sf.basedb.core.plugin.InteractivePlugin;
import net.sf.basedb.core.plugin.About;
import net.sf.basedb.core.plugin.AboutImpl;
import net.sf.basedb.core.plugin.GuiContext;
import net.sf.basedb.core.plugin.Request;
import net.sf.basedb.core.plugin.Response;
import net.sf.basedb.plugins.AbstractFlatFileImporter;
import net.sf.basedb.util.parser.FlatFileParser;
import net.sf.basedb.util.parser.Mapper;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Collections;
import java.util.Set;
import java.util.TreeSet;
import java.util.StringTokenizer;
import java.util.Iterator;

/**
	A plugin that imports features to an ArrayDesign from a gal file. The ArrayDesign
	will not have any connections to plates.
	
	@base.modified $Date: 2006-05-24 11:50:59 +0200 (Wed, 24 May 2006) $
	@author Enell
	@version 2.0
*/
public class GalFileImporter
	extends AbstractFlatFileImporter
	implements InteractivePlugin
{

	private static final About about = 
		new AboutImpl
		(
			"Gal File importer",
			"This plugin is used to import features to an ArrayDesign from a GAL file.",
			"2.0",
			"2006, FHCRC",
			null,
			"base@thep.lu.se",
			"http://base.thep.lu.se"
		);

	private static final Set<GuiContext> guiContexts = 
		Collections.singleton(new GuiContext(Item.ARRAYDESIGN, GuiContext.Type.ITEM));

	private static final StringParameterType requiredColumnMapping = new StringParameterType(255, null, true);
	private static final StringParameterType optionalColumnMapping = new StringParameterType(255, null, false);
	

	private static final PluginParameter<String> reporterIdColumnMapping = new PluginParameter<String>(
		"reporterIdColumnMapping",
		"Reporter ID",
		"Mapping that picks the reporter's ID from the data columns. " +
		"For example: \\ID\\",
		requiredColumnMapping
		);

	private static final PluginParameter<String> blockColumnMapping = new PluginParameter<String>(
		"blockColumnMapping",
		"Block",
		"Mapping that picks the feature's block number from the data columns. "+
		"You must specify either this mapping or mappings for the meta coordinates. " +
		"Example: \\Block\\",
		optionalColumnMapping
		);

	private static final PluginParameter<String> columnColumnMapping = new PluginParameter<String>(
		"columnColumnMapping",
		"Column",
		"Mapping that picks the feature's column position in a block from the data columns. " +
		"For example: \\Column\\",
		requiredColumnMapping
		);
	
	private static final PluginParameter<String> rowColumnMapping = new PluginParameter<String>(
		"rowColumnMapping",
		"Row",
		"Mapping that picks the feature's row position in a block from the data columns. " +
		"For example: \\Row\\",
		requiredColumnMapping
		);
	
	private RequestInformation configurePlugin;
	private RequestInformation configureJob;
	
	private DbControl dc;
	private FeatureBatcher batcher;
	private ReporterBatcher reporterBatcher;
	private ArrayDesign arrayDesign;
	private Map<String, Mapper> columnMappings;
	private Map<Integer, ArrayDesignBlock> blocks;
	private FlatFileParser ffp;

	private ItemParameterType<ArrayDesign> arrayDesignType;
	private PluginParameter<ArrayDesign> arrayDesignParameter;
	
	private List<PluginParameter<String>> allColumnMappings;
	
	private int numFeatures;
	private int numBlocks;
	private ArrayLayout arrayLayout; // used to figure out where each block is (metaX and metaY)
	
	/**
		Create a new importer.
	*/
	public GalFileImporter()
	{
		arrayLayout = new ArrayLayout();
	}
	
	/*
		From the Plugin interface
		-------------------------------------------
	*/
	public About getAbout()
	{
		return about;
	}
	// -------------------------------------------
	
	/*
		From the InteractivePlugin interface
		-------------------------------------------
	*/
	/**
		Return a set containing the element [ARRAYDESIGN, ITEM].
	*/
	public Set<GuiContext> getGuiContexts()
	{
		return guiContexts;
	}

	/**
		Returns null if the item is a {@link ArrayDesign} that doesn't already have 
		any features and isn't an affy design.
	*/
	public String isInContext(GuiContext context, Object item)
	{
		String message = null;
		if (item == null)
		{
			message = "The object is null";
		}
		else if (!(item instanceof ArrayDesign))
		{
			message = "The object is not an ArrayDesign: " + item;
		}
		else
		{
			ArrayDesign ad = (ArrayDesign)item;
			if (ad.hasFeatures())
			{
				message = "The array design already has features: " + ad.getName();
			}
			else if (ad.isAffyChip())
			{
				message = "Affy array designs are not supported: " + ad.getName();
			}
		}
		return message;
	}
	/**
		The {@link Request#COMMAND_CONFIGURE_PLUGIN} command will ask for
		parser regular expressions and column mappings.
		The {@link Request#COMMAND_CONFIGURE_JOB} command will ask for
		a file and the array design that features should be added to.
	*/
	public RequestInformation getRequestInformation(GuiContext context, String command) 
		throws BaseException
	{
		RequestInformation requestInformation = null;
		if (command.equals(Request.COMMAND_CONFIGURE_PLUGIN))
		{
			requestInformation = getConfigurePluginParameters(context);
		}
		else if (command.equals(Request.COMMAND_CONFIGURE_JOB))
		{
			requestInformation = getConfigureJobParameters(context);
			ItemContext fileContext = sc.getCurrentContext(Item.FILE);
			fileContext.setPropertyFilter(FileType.getPropertyFilter(FileType.REPORTER_MAP));
			fileContext.setPropertyFilter(getPrimaryLocationFilter());
		}
		return requestInformation;
	}
	/**
		Store configuration settings for {@link Request#COMMAND_CONFIGURE_PLUGIN} and
		{@link Request#COMMAND_CONFIGURE_JOB}.
	*/
	public void configure(GuiContext context, Request request, Response response)
	{
		String command = request.getCommand();
		try
		{
			if (command.equals(Request.COMMAND_CONFIGURE_PLUGIN))
			{

				List<Throwable> errors = 
					validateRequestParameters(getConfigurePluginParameters(context).getParameters(), request);
				if (errors != null)
				{
					response.setError(errors.size() + " invalid parameter(s) were found in the request", errors);
					return;
				}
				
				// Check that either block or meta coordinates have been mapped
				String blockMapping = (String)request.getParameterValue("blockColumnMapping");
				String metaGridXMapping = (String)request.getParameterValue("metaGridXColumnMapping");
				String metaGridYMapping = (String)request.getParameterValue("metaGridYColumnMapping");
				if (blockMapping == null && (metaGridXMapping == null || metaGridYMapping == null))
				{
					response.setError("You must at least map either block or meta coordinates to columns.", null);
					return;
				}
				
				// Parser settings
				storeValue(configuration, request, headerRegexpParameter);
				storeValue(configuration, request, dataHeaderRegexpParameter);
				storeValue(configuration, request, dataSplitterRegexpParameter);
				storeValue(configuration, request, trimQuotesParameter);
				storeValue(configuration, request, ignoreRegexpParameter);
				storeValue(configuration, request, dataFooterRegexpParameter);
				storeValue(configuration, request, minDataColumnsParameter);
				storeValue(configuration, request, maxDataColumnsParameter);

				// Column mappings
				for (PluginParameter<?> pp : getAllColumnMappings())
				{
					storeValue(configuration, request, pp);
				}

				response.setDone("Plugin configuration complete");
			}
			else if (command.equals(Request.COMMAND_CONFIGURE_JOB))
			{
				List<Throwable> errors = 
					validateRequestParameters(getConfigureJobParameters(context).getParameters(), request);
				if (errors != null)
				{
					response.setError(errors.size() + " invalid parameter(s) were found in the request", errors);
					return;
				}
				storeValue(job, request, fileParameter);
				storeValue(job, request, arrayDesignParameter);
				response.setDone("Job configuration complete", Job.ExecutionTime.SHORT);
				// TODO - maybe check file size to make a better estimate
			}
		}
		catch (Throwable ex)
		{
			response.setError(ex.getMessage(), Arrays.asList(ex));
		}
	}
	// -------------------------------------------

	/*
		From the AbstractFlatFileReporter class
		-------------------------------------------
	*/
	/**
		Create a {@link DbControl} and a {@link FeatureBatcher}. 
		Check that the {@link ArrayDesign} doesn't already have features.
	*/
	@Override
	protected void begin(FlatFileParser ffp)
		throws BaseException
	{
		super.begin(ffp);
		dc = sc.newDbControl();
		arrayDesign = (ArrayDesign)job.getValue("arrayDesign");
		dc.reattachItem(arrayDesign);
		if (arrayDesign.hasFeatures())
		{
			throw new PermissionDeniedException("The array design already has features. "+arrayDesign.getName()+"["+arrayDesign.getId()+"]");
		}
		batcher = arrayDesign.getFeatureBatcher();
		reporterBatcher = ReporterBatcher.getNew(dc);
		this.ffp = ffp;
		numFeatures = 0;
		numBlocks = 0;
		blocks = new HashMap<Integer, ArrayDesignBlock>();
	}

	/**
		Initialise column <code>Mapper</code>:s.
	*/
	@Override
	protected void beginData()
	{
		columnMappings = new HashMap<String, Mapper>();
		for (PluginParameter<?> pp : getAllColumnMappings())
		{
			columnMappings.put(pp.getName(), ffp.getMapper((String)configuration.getValue(pp.getName())));
		}
	}	
	
	
	@Override
	protected void handleHeader(FlatFileParser.Line line)
		throws BaseException
	{
		String lineS = line.line();
		System.err.println("Parsing header line");
		System.err.println(lineS);
		lineS = lineS.replaceAll("\"","");
		if (lineS.matches("Block\\d+.*")){
System.out.println("match " + lineS);
			handleNewBlockLine(lineS);
		}else if (true){
System.out.println("didn't match but... " + lineS);
			handleNewBlockLine(lineS);
		}else{
System.out.println("No match " + lineS);
		}
	
	}
	/** line will look like
	<pre>
	"Block22=	5000	23000	100	22	200	22	200"
	or 
	Block22=5000,23000,100,22,200,22,200
	</pre>
	*/
	
	/*
	* The metaX and metaY positions are not specified specifically so we have to infer them from 
	* the oriX and oriY so we have to keep track of them as we make them
	*/
	private void handleNewBlockLine(String line){
System.out.println("handleNewBlockLine " + line);
		line = line.replaceAll("\"","");
		String aLine = line.substring(5);
		int indexE = aLine.indexOf("=");
		String blockNumStr = aLine.substring(0,indexE);
		String nLine = aLine.substring(indexE + 1);
		StringTokenizer tok;
		if (nLine.indexOf(",") > -1 ){
			tok = new StringTokenizer(nLine,",");
		}else{
			tok = new StringTokenizer(nLine);
		}
		Integer block = new Integer(blockNumStr);
		Integer oriX = new Integer((String)tok.nextElement());
		Integer oriY = new Integer((String)tok.nextElement());
		int diam = Integer.parseInt((String)tok.nextElement());
		int numX = Integer.parseInt((String)tok.nextElement());
		int spacingX = Integer.parseInt((String)tok.nextElement());
		int numY = Integer.parseInt((String)tok.nextElement());
		int spacingY = Integer.parseInt((String)tok.nextElement());
		
		Integer[] metaPosition = arrayLayout.getLayoutPostion(oriX,oriY);
		Integer metaGridX = metaPosition[0];
		Integer metaGridY = metaPosition[1];
		
		BlockInfo bi;
		bi = new BlockInfo(block, metaGridX, metaGridY);
		ArrayDesignBlock adb = arrayDesign.addArrayDesignBlock(bi);
		adb.setSpacingX(spacingX);
		adb.setSpacingY(spacingY);
		adb.setBlockSizeX(numX);
		adb.setBlockSizeY(numY);
		blocks.put(block,adb);
System.out.println("adding block " + block);
		numBlocks++;
	}
	
	@Override
	protected void handleData(FlatFileParser.Data data) 
		throws BaseException
	{
		String externalId = columnMappings.get("reporterIdColumnMapping").getValue(data);
		if (externalId.equals("EMPTY")) return;
		ReporterData reporter = externalId == null ? null : reporterBatcher.getByExternalId(externalId);
		Integer block = columnMappings.get("blockColumnMapping").getInt(data);
		ArrayDesignBlock adb = blocks.get(block);
		
		if (adb == null){
			throw new InvalidDataException("ArrayDesign has no block " + block);
		}
		FeatureData feature = batcher.newFeature(adb, reporter);
		try
		{
			feature.setRow(Integer.parseInt(columnMappings.get("rowColumnMapping").getValue(data)));
			feature.setColumn(Integer.parseInt(columnMappings.get("columnColumnMapping").getValue(data)));
		}
		catch(NumberFormatException e)
		{
			throw new InvalidDataException("Row or column at line "+data.lineNo()+" is not a number");
		}
		batcher.insert(feature);
		numFeatures++;
	}

	/**
		Close and commit/rollback the FeatureBatcher and DbControl.
	*/
	@Override
	protected void end(boolean success)
		throws BaseException
	{
		blocks.clear();
		try
		{
			batcher.close();
			reporterBatcher.close();
			if (success)
			{
				dc.commit();
			}
			else
			{
				dc.close();
			}
		}
		catch (BaseException ex)
		{
			dc.close();
			throw ex;
		}
		finally
		{
			super.end(success);
		}
	}
	/**
		Return <code>x features inserted; y blocks inserted</code>.
	*/
	protected String getSuccessMessage()
	{
		return numFeatures + (numFeatures == 1 ? " feature inserted; " : " features inserted; ")
			+ numBlocks + (numBlocks == 1 ? " block inserted; " : " blocks inserted");
	}
	// -------------------------------------------

	private List<PluginParameter<String>> getAllColumnMappings()
	{
		if (allColumnMappings == null)
		{
			allColumnMappings = new ArrayList<PluginParameter<String>>();
			allColumnMappings.add(reporterIdColumnMapping);
			allColumnMappings.add(blockColumnMapping);
			allColumnMappings.add(columnColumnMapping);
			allColumnMappings.add(rowColumnMapping);
		}
		return allColumnMappings;
	}
	
	private RequestInformation getConfigureJobParameters(GuiContext context)
	{
		if (configureJob == null)
		{
			arrayDesignType = new ItemParameterType<ArrayDesign>(ArrayDesign.class, null, true, 1, null);
			arrayDesignParameter = new PluginParameter<ArrayDesign>(
				"arrayDesign",
				"Array Design",
				"The array design assigned to the imported features",
				arrayDesignType);
			
			// Parameters for CONFIGURE_JOB
			List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
			parameters.add(arrayDesignParameter);
			parameters.add(fileParameter);
			
			configureJob = new RequestInformation
			(
				Request.COMMAND_CONFIGURE_JOB,
				"Select file to import features from",
				"Here you select which file to import the features from.",
				parameters
			);
		}
		return configureJob;
	}

	private RequestInformation getConfigurePluginParameters(GuiContext context)
	{
		if (configurePlugin == null)
		{
			// Parameters object for CONFIGURE_PLUGIN
			List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();

			// Parser regular expressions
			parameters.add(parserSection);
			parameters.add(headerRegexpParameter);
			parameters.add(dataHeaderRegexpParameter);
			parameters.add(dataSplitterRegexpParameter);
			parameters.add(trimQuotesParameter);
			parameters.add(ignoreRegexpParameter);
			parameters.add(dataFooterRegexpParameter);
			parameters.add(minDataColumnsParameter);
			parameters.add(maxDataColumnsParameter);
			
			// Column mappings
			parameters.add(mappingSection);
			parameters.addAll(getAllColumnMappings());

			configurePlugin = new RequestInformation
			(
				Request.COMMAND_CONFIGURE_PLUGIN,
				"Parser settings",
				"Enter the regular expressions used to parse the text file " +
				"and the column mappings used to find matching properties for the features.",
				parameters
			);
		}
		return configurePlugin;
	}
	
	/** this allow us to find metaX and metaY based on the oriX and oriY. This will only
	* work if the blocks are added in order for either rows or columns.
	*/
	class ArrayLayout{
		TreeSet<Integer> xLayout;
		TreeSet<Integer> yLayout;
		
		ArrayLayout(){
			xLayout = new TreeSet<Integer>();
			yLayout = new TreeSet<Integer>();
		}
		
		Integer[] getLayoutPostion(Integer oriX, Integer oriY){
			xLayout.add(oriX);
System.out.println("add " + oriX + " to the treeset X");
			yLayout.add(oriY);
System.out.println("add " + oriY + " to the treeset Y");
			int xx = getIndex(xLayout,oriX);
			int yy = getIndex(yLayout,oriY);
			return new Integer[] {new Integer(xx),new Integer(yy)};
		}
		
		/** returns a 1 based index of the Integer in the sorted set, returns -1 if not found*/
		int getIndex(TreeSet<Integer> set,Integer testI){
System.out.println("getIndex " + testI);

			int i = 0;
			for (Iterator it = set.iterator();it.hasNext();){
				i++;
				Integer value = (Integer)it.next();
System.out.println(i + " testing " + value + " vs " + testI);
				if (value.equals(testI)) return i;
			}
System.out.println("Could not find " + testI + " in the treeset");
			return -1;
		}
	}	
			
}