SqlLoaderControlParserImpl

/*
 *
 * The DbUnit Database Testing Framework
 * Copyright (C)2002-2008, DbUnit.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */
package org.dbunit.dataset.sqlloader;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.net.URL;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.text.CharacterIterator;
import java.text.StringCharacterIterator;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.dbunit.dataset.common.handlers.EscapeHandler;
import org.dbunit.dataset.common.handlers.IllegalInputCharacterException;
import org.dbunit.dataset.common.handlers.IsAlnumHandler;
import org.dbunit.dataset.common.handlers.Pipeline;
import org.dbunit.dataset.common.handlers.PipelineException;
import org.dbunit.dataset.common.handlers.QuoteHandler;
import org.dbunit.dataset.common.handlers.SeparatorHandler;
import org.dbunit.dataset.common.handlers.TransparentHandler;
import org.dbunit.dataset.common.handlers.WhitespacesHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Parser which parses Oracle SQLLoader files.
 *
 * @author Stephan Strittmatter (stritti AT users.sourceforge.net)
 * @author Last changed by: $Author$
 * @version $Revision$ $Date$
 * @since 2.4.0
 */
public class SqlLoaderControlParserImpl implements SqlLoaderControlParser {

    public static final char SEPARATOR_CHAR = ';';

    /** The pipeline. */
    private Pipeline pipeline;

    private String tableName;

//    private String fieldTerminator;
//
//    private String fieldEnclosure;

    private boolean hasTrailingNullCols;

    /**
     * Logger for this class
     */
    private static final Logger logger = LoggerFactory.getLogger(SqlLoaderControlParserImpl.class);


    /**
     * The Constructor.
     */
    public SqlLoaderControlParserImpl() {

        resetThePipeline();

    }

    /**
     * Reset the pipeline.
     */
    private void resetThePipeline() {
        logger.debug("resetThePipeline() - start");

        this.pipeline = new Pipeline();
        this.pipeline.getPipelineConfig().setSeparatorChar(SEPARATOR_CHAR);

        //TODO add this.fieldEnclosure
        getPipeline().putFront(SeparatorHandler.ENDPIECE());
        getPipeline().putFront(EscapeHandler.ACCEPT());
        getPipeline().putFront(IsAlnumHandler.QUOTE());
        getPipeline().putFront(QuoteHandler.QUOTE());
        getPipeline().putFront(EscapeHandler.ESCAPE());
        getPipeline().putFront(WhitespacesHandler.IGNORE());
        getPipeline().putFront(TransparentHandler.IGNORE());

    }

    /**
     * Parse.
     *
     * @param csv the csv
     *
     * @return the list
     *
     * @throws IllegalInputCharacterException the illegal input character exception
     * @throws PipelineException the pipeline exception
     *
     * @see org.dbunit.dataset.sqlloader.SqlLoaderControlParser#parse(java.lang.String)
     */
    public List parse(String csv) throws PipelineException, IllegalInputCharacterException {
        logger.debug("parse(csv={}) - start", csv);

        getPipeline().resetProducts();
        CharacterIterator iterator = new StringCharacterIterator(csv);
        for (char c = iterator.first(); c != CharacterIterator.DONE; c = iterator.next()) {
            getPipeline().handle(c);
        }
        getPipeline().noMoreInput();
        getPipeline().thePieceIsDone();

        return getPipeline().getProducts();
    }

    /**
     * Parse.
     *
     * @param url the URL
     *
     * @return the list
     *
     * @throws IOException the IO exception
     * @throws SqlLoaderControlParserException the oracle control parser exception
     *
     * @see org.dbunit.dataset.sqlloader.SqlLoaderControlParser#parse(java.net.URL)
     */
    public List parse(URL url) throws IOException, SqlLoaderControlParserException {
        logger.debug("parse(url={}) - start", url);
        return parse(new File(url.toString()));
    }

    /**
     * Parse.
     *
     * @param controlFile the source
     *
     * @return the list of column names as Strings
     *
     * @throws IOException the IO exception
     * @throws SqlLoaderControlParserException the oracle control parser exception
     * @see org.dbunit.dataset.sqlloader.SqlLoaderControlParser#parse(java.io.File)
     */
    public List parse(File controlFile)
    throws IOException, SqlLoaderControlParserException
    {
        logger.debug("parse(controlFile={}) - start", controlFile);

        FileInputStream fis = new FileInputStream(controlFile);

        FileChannel fc = fis.getChannel();

        MappedByteBuffer mbf = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
        byte[] barray = new byte[(int) (fc.size())];
        mbf.get(barray);

        String lines = new String(barray); //one big string

        lines = lines.replaceAll("\r", ""); //unify to UNIX style to have easier regexp transformations.

        if (parseForRegexp(lines, "(LOAD\\sDATA).*") != null) {

        	String fileName = parseForRegexp(lines, ".*INFILE\\s'(.*?)'.*");
        	File dataFile = resolveFile(controlFile.getParentFile(), fileName);

            this.tableName = parseForRegexp(lines, ".*INTO\\sTABLE\\s(.*?)\\s.*");

//            this.fieldTerminator = parseForRegexp(lines, ".*TERMINATED BY [\"|'](.*?)[\"|'].*");
//
//            this.fieldEnclosure = parseForRegexp(lines, ".*OPTIONALLY ENCLOSED BY '(.*?)'.*");

            if (parseForRegexp(lines, ".*(TRAILING NULLCOLS).*") != "") {
                this.hasTrailingNullCols = true;
            }
            else {
                this.hasTrailingNullCols = false;
            }

            List rows = new ArrayList();
            List columnList = parseColumns(lines, rows);

            LineNumberReader lineNumberReader =
                new LineNumberReader(new InputStreamReader(new FileInputStream(dataFile)));
            try {
                parseTheData(columnList, lineNumberReader, rows);
            }
            finally {
                lineNumberReader.close();
            }

            return rows;
        }
        else {
            throw new SqlLoaderControlParserException("Control file "
                    + controlFile + " not starting using 'LOAD DATA'");
        }
    }

    private File resolveFile(File parentDir, String fileName) {
    	// Initially assume that we have an absolute fileName
    	File dataFile = new File(fileName);

    	// If fileName was not absolute build it using the given parent
    	if(!dataFile.isAbsolute()) {
    		fileName = fileName.replaceAll("\\\\", "/");
    		// remove "./" characters from name at the beginning if needed
    		if(fileName.startsWith("./")){
    			fileName = fileName.substring(2);
    		}
    		// remove "." character from name at the beginning if needed
    		if(fileName.startsWith(".")){
    			fileName = fileName.substring(1);
    		}
    		dataFile = new File(parentDir, fileName);
    	}
    	return dataFile;
	}

	protected String parseForRegexp(String controlFileContent, String regexp)
    throws IOException
    {
        logger.debug("parseForRegexp(controlFileContent={}, regexp={}) - start", controlFileContent, regexp);

        if (controlFileContent == null) {
            throw new NullPointerException("control file has no content");
        }

        final Pattern pattern = Pattern.compile(regexp, Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
        final Matcher matches = pattern.matcher(controlFileContent);

        if (matches.find()) {
            String inFileLine = matches.group(1);

            return inFileLine;
        }
        else {
            return null;
        }
    }

    /**
     * parse the first line of data from the given source.
     *
     * @param rows the rows
     * @param lineNumberReader the line number reader
     * @param controlFile the source
     *
     * @return the list of column names as Strings
     *
     * @throws IOException the IO exception
     * @throws SqlLoaderControlParserException the oracle control parser exception
     */
    private List parseColumns(String controlFileContent, List rows) throws IOException,
    SqlLoaderControlParserException
    {
        logger.debug("parseColumns(controlFileContent={}, rows={}) - start", controlFileContent, rows);

        List columnList;

        final Pattern pattern =
            Pattern
            .compile(".*FIELDS\\s.*\\(\\n(.*?)\\n\\)", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
        final Matcher matches = pattern.matcher(controlFileContent);

        if (matches.find()) {
            String columnFragment = matches.group(1);
            //firstLine = firstLine.replaceAll("(\n|\r)", "");

            columnList = new ArrayList();

            columnFragment = columnFragment.replaceAll("\".*?\"", "");
            columnFragment = columnFragment.replaceAll("\n", "");

            StringTokenizer tok = new StringTokenizer(columnFragment, ",");

            while (tok.hasMoreElements()) {

                String col = (String) tok.nextElement();
                col = parseForRegexp(col, ".*^([a-zA-Z0-9_]*)\\s").trim(); //column is the first part.
                columnList.add(col);
            }

            //columnsInFirstLine = parse(firstLine);
            rows.add(columnList);
        }

        else {
            columnList = null;
        }

        return columnList;
    }

    /**
     * Parses the the data.
     *
     * @param rows the rows
     * @param columnList the columns in first line
     * @param lineNumberReader the line number reader
     *
     * @throws IOException the IO exception
     * @throws SqlLoaderControlParserException the oracle control parser exception
     */
    private void parseTheData(final List columnList, LineNumberReader lineNumberReader, List rows)
    throws IOException, SqlLoaderControlParserException
    {
        if(logger.isDebugEnabled())
            logger.debug("parseTheData(columnList={}, lineNumberReader={}, rows={}) - start",
                    new Object[] {columnList, lineNumberReader, rows} );

        int nColumns = columnList.size();
        List columns;
        while ((columns = collectExpectedNumberOfColumns(nColumns, lineNumberReader)) != null) {
            rows.add(columns);
        }
    }

    /**
     * Collect expected number of columns.
     *
     * @param expectedNumberOfColumns the expected number of columns
     * @param lineNumberReader the line number reader
     *
     * @return the list
     *
     * @throws IOException the IO exception
     * @throws SqlLoaderControlParserException the oracle control parser exception
     */
    private List collectExpectedNumberOfColumns(
            int expectedNumberOfColumns,
            LineNumberReader lineNumberReader) throws IOException, SqlLoaderControlParserException
    {
        if(logger.isDebugEnabled())
            logger.debug("collectExpectedNumberOfColumns(expectedNumberOfColumns={}, lineNumberReader={}) - start",
                String.valueOf(expectedNumberOfColumns), lineNumberReader);

        String anotherLine = lineNumberReader.readLine();
        if (anotherLine == null) {
            return null;
        }

        List columns = null;
        int columnsCollectedSoFar = 0;
        final StringBuilder buffer = new StringBuilder();
        boolean shouldProceed = false;
        while (columnsCollectedSoFar < expectedNumberOfColumns) {
            try {
                buffer.append(anotherLine);
                columns = parse(buffer.toString());
                columnsCollectedSoFar = columns.size();
            }
            catch (IllegalStateException e) {
                resetThePipeline();
                anotherLine = lineNumberReader.readLine();
                if (anotherLine == null) {
                    break;
                }
                buffer.append("\n");
                shouldProceed = true;
            }
            if (!shouldProceed) {
                break;
            }
        }

        if (columnsCollectedSoFar != expectedNumberOfColumns) {
            if (this.hasTrailingNullCols) {
                columns.add(SqlLoaderControlProducer.NULL);
            }
            else {

                String message =
                    new StringBuilder("Expected ")
                        .append(expectedNumberOfColumns).append(" columns on line ")
                        .append(lineNumberReader.getLineNumber()).append(", got ")
                        .append(columnsCollectedSoFar).append(". Offending line: ").append(buffer)
                        .toString();
                throw new SqlLoaderControlParserException(message);
            }
        }
        return columns;
    }

    /**
     * Gets the pipeline.
     *
     * @return the pipeline
     */
    Pipeline getPipeline()
    {
        return this.pipeline;
    }

    /**
     * Sets the pipeline.
     *
     * @param pipeline the pipeline
     */
    void setPipeline(Pipeline pipeline)
    {
        this.pipeline = pipeline;
    }

    public String getTableName()
    {
        return this.tableName;
    }
}