FlatXmlProducer.java

/*
 *
 * The DbUnit Database Testing Framework
 * Copyright (C)2002-2004, DbUnit.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */
package org.dbunit.dataset.xml;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;

import org.dbunit.dataset.Column;
import org.dbunit.dataset.DataSetException;
import org.dbunit.dataset.DefaultTableMetaData;
import org.dbunit.dataset.IDataSet;
import org.dbunit.dataset.ITableMetaData;
import org.dbunit.dataset.NoSuchColumnException;
import org.dbunit.dataset.OrderedTableNameMap;
import org.dbunit.dataset.datatype.DataType;
import org.dbunit.dataset.stream.BufferedConsumer;
import org.dbunit.dataset.stream.DefaultConsumer;
import org.dbunit.dataset.stream.IDataSetConsumer;
import org.dbunit.dataset.stream.IDataSetProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;

/**
 * @author Manuel Laflamme
 * @author Last changed by: $Author$
 * @version $Revision$ $Date$
 * @since 1.5 (Apr 18, 2003)
 */
public class FlatXmlProducer extends DefaultHandler implements IDataSetProducer, ContentHandler
{

    /**
     * Logger for this class
     */
    private static final Logger logger = LoggerFactory.getLogger(FlatXmlProducer.class);

    private static final IDataSetConsumer EMPTY_CONSUMER = new DefaultConsumer();
    private static final String DATASET = "dataset";

    private final InputSource _inputSource;
    private final EntityResolver _resolver;
    private boolean _validating = false;
    /**
     * The dataset used to retrieve the metadata for the tables via {@link IDataSet#getTableMetaData(String)}. 
     * Can be null
     */
    private IDataSet _metaDataSet;
    /**
     * The DTD handler which is used to parse a DTD if available. The result of the parsing is stored in
     * {@link #_metaDataSet}.
     */
    private FlatDtdHandler _dtdHandler;
    
    /**
     * The current line number in the current table
     */
    private int _lineNumber = 0;
    /**
     * The current line number
     */
    private int _lineNumberGlobal = 0;
    /**
     * Whether the column sensing feature should be used to dynamically recognize new columns
     * during the parse process.
     */
    private boolean _columnSensing = false;
    private boolean _caseSensitiveTableNames;

    /**
     * The consumer which is responsible for creating the datasets and tables
     */
    private IDataSetConsumer _consumer = EMPTY_CONSUMER;
    /**
     * The ordered table name map which also holds the currently active {@link ITableMetaData}
     */
    private OrderedTableNameMap _orderedTableNameMap;

    
    public FlatXmlProducer(InputSource xmlSource)
    {
        this(xmlSource, true);
    }

    public FlatXmlProducer(InputSource xmlSource, boolean dtdMetadata)
    {
        this(xmlSource, dtdMetadata, false);
    }

    public FlatXmlProducer(InputSource xmlSource, IDataSet metaDataSet)
    {
        _inputSource = xmlSource;
        _metaDataSet = metaDataSet;
        _resolver = this;
        _caseSensitiveTableNames = metaDataSet.isCaseSensitiveTableNames();
        initialize(false);
    }

    public FlatXmlProducer(InputSource xmlSource, EntityResolver resolver)
    {
        _inputSource = xmlSource;
        _resolver = resolver;
        initialize(true);
    }
    
    /**
     * @param xmlSource The input datasource
     * @param dtdMetadata Whether or not DTD metadata is available to parse via a DTD handler
     * @param columnSensing Whether or not the column sensing feature should be used (see FAQ)
     */
    public FlatXmlProducer(InputSource xmlSource, boolean dtdMetadata, boolean columnSensing)
    {
        this(xmlSource, dtdMetadata, columnSensing, false);
    }
    
    /**
     * @param xmlSource The input datasource
     * @param dtdMetadata Whether or not DTD metadata is available to parse via a DTD handler
     * @param columnSensing Whether or not the column sensing feature should be used (see FAQ)
     * @param caseSensitiveTableNames Whether or not this dataset should use case sensitive table names
     * @since 2.4.2
     */
    public FlatXmlProducer(InputSource xmlSource, boolean dtdMetadata, boolean columnSensing, boolean caseSensitiveTableNames)
    {
        _inputSource = xmlSource;
        _columnSensing = columnSensing;
        _caseSensitiveTableNames = caseSensitiveTableNames;
        _resolver = this;
        initialize(dtdMetadata);
    }


    
    private void initialize(boolean dtdMetadata)
    {
        if (dtdMetadata)
        {
            this._dtdHandler = new FlatDtdHandler(this);
        }
    }
    
    /**
     * @return Whether or not this producer works case sensitively
     * @since 2.4.7
     */
    public boolean isCaseSensitiveTableNames() {
        return _caseSensitiveTableNames;
    }

    private ITableMetaData createTableMetaData(String tableName, Attributes attributes) throws DataSetException
    {
        if (logger.isDebugEnabled())
            logger.debug("createTableMetaData(tableName={}, attributes={}) - start", tableName, attributes);

        // First try to find it in the DTD's dataset
        if (_metaDataSet != null)
        {
            return _metaDataSet.getTableMetaData(tableName);
        }

        // Create metadata from attributes
        Column[] columns = new Column[attributes.getLength()];
        for (int i = 0; i < attributes.getLength(); i++)
        {
            columns[i] = new Column(attributes.getQName(i), DataType.UNKNOWN);
        }

        return new DefaultTableMetaData(tableName, columns);
    }
    
    
    /**
     * merges the existing columns with the potentially new ones.
     * @param columnsToMerge List of extra columns found, which need to be merge back into the metadata.
     * @return ITableMetaData The merged metadata object containing the new columns
     * @throws DataSetException
     */
    private ITableMetaData mergeTableMetaData(List columnsToMerge, ITableMetaData originalMetaData) throws DataSetException
    {
        Column[] columns = new Column[originalMetaData.getColumns().length + columnsToMerge.size()];
        System.arraycopy(originalMetaData.getColumns(), 0, columns, 0, originalMetaData.getColumns().length);
        
        for (int i = 0; i < columnsToMerge.size(); i++)
        {
            Column column = (Column)columnsToMerge.get(i);
            columns[columns.length - columnsToMerge.size() + i] = column;
        }
        
        return new DefaultTableMetaData(originalMetaData.getTableName(), columns);
    }
    
    /**
     * @return The currently active table metadata or <code>null</code> if no active
     * metadata exists.
     */
    private ITableMetaData getActiveMetaData()
    {
        if(_orderedTableNameMap != null)
        {
            String lastTableName = _orderedTableNameMap.getLastTableName();
            if(lastTableName != null)
            {
                return (ITableMetaData) _orderedTableNameMap.get(lastTableName);
            }
            else
            {
                return null;
            }
        }
        else
        {
            return null;
        }
        
    }
    
    /**
     * @param tableName
     * @return <code>true</code> if the given tableName is a new one
     * which means that it differs from the last active table name.
     */
    private boolean isNewTable(String tableName) 
    {
        return !_orderedTableNameMap.isLastTable(tableName);
    }


    /**
     * parses the attributes in the current row, and checks whether a new column 
     * is found.
     * 
     * <p>Depending on the value of the <code>columnSensing</code> flag, the appropriate
     * action is taken:</p>
     * 
     * <ul>
     *   <li>If it is true, the new column is merged back into the metadata;</li>
     *   <li>If not, a warning message is displayed.</li>
     * </ul>
     * 
     * @param attributes  Attributed for the current row.
     * @throws DataSetException
     */
    protected void handleMissingColumns(Attributes attributes)
            throws DataSetException
    {
        List columnsToMerge = new ArrayList();
        
        ITableMetaData activeMetaData = getActiveMetaData();
        // Search all columns that do not yet exist and collect them
        int attributeLength = attributes.getLength();
        for (int i = 0 ; i < attributeLength; i++)
        {
            try {
                activeMetaData.getColumnIndex(attributes.getQName(i));
            } 
            catch (NoSuchColumnException e) {
                columnsToMerge.add(new Column(attributes.getQName(i), DataType.UNKNOWN));
            }
        }
        
        if (!columnsToMerge.isEmpty())
        {
            if (_columnSensing)
            {
                logger.debug("Column sensing enabled. Will create a new metaData with potentially new columns if needed");
                activeMetaData = mergeTableMetaData(columnsToMerge, activeMetaData);
                _orderedTableNameMap.update(activeMetaData.getTableName(), activeMetaData);
                // We also need to recreate the table, copying the data already collected from the old one to the new one
                _consumer.startTable(activeMetaData);
            } 
            else
            {
                final StringBuilder extraColumnNames = new StringBuilder();
                for (Iterator i = columnsToMerge.iterator(); i.hasNext();) {
                    Column col = (Column) i.next();
                    extraColumnNames.append(extraColumnNames.length() > 0 ? "," : "").append(col.getColumnName());
                }
                String msg = "Extra columns (" + extraColumnNames.toString() + ") on line " + (_lineNumber + 1)
                        + " for table " + activeMetaData.getTableName() + " (global line number is "
                        + _lineNumberGlobal + "). Those columns will be ignored.";
                msg += "\n\tPlease add the extra columns to line 1,"
                    + " or use a DTD to make sure the value of those columns are populated" 
                    + " or specify 'columnSensing=true' for your FlatXmlProducer.";
                msg += "\n\tSee FAQ for more details.";
                logger.warn(msg);
            }
        }
    }
    
    public void setColumnSensing(boolean columnSensing)
    {
        _columnSensing = columnSensing;
    }

    public void setValidating(boolean validating)
    {
        _validating = validating;
    }

    ////////////////////////////////////////////////////////////////////////////
    // IDataSetProducer interface

    public void setConsumer(IDataSetConsumer consumer) throws DataSetException
    {
        logger.debug("setConsumer(consumer) - start");

        if(this._columnSensing) {
            _consumer = new BufferedConsumer(consumer);
        }
        else {
            _consumer = consumer;
        }
    }

    public void produce() throws DataSetException
    {
        logger.debug("produce() - start");

        try
        {
            SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
            saxParserFactory.setValidating(_validating);
            XMLReader xmlReader = saxParserFactory.newSAXParser().getXMLReader();

            if(_dtdHandler != null)
            {
                FlatDtdHandler.setLexicalHandler(xmlReader, _dtdHandler);
                FlatDtdHandler.setDeclHandler(xmlReader, _dtdHandler);
            }

            xmlReader.setContentHandler(this);
            xmlReader.setErrorHandler(this);
            xmlReader.setEntityResolver(_resolver);
            xmlReader.parse(_inputSource);
        }
        catch (ParserConfigurationException e)
        {
            throw new DataSetException(e);
        }
        catch (SAXException e)
        {
            DataSetException exceptionToRethrow = XmlProducer.buildException(e);
            throw exceptionToRethrow;
        }
        catch (IOException e)
        {
            throw new DataSetException(e);
        }
    }

    ////////////////////////////////////////////////////////////////////////////
    // EntityResolver interface

    public InputSource resolveEntity(String publicId, String systemId)
            throws SAXException
    {
        logger.debug("resolveEntity(publicId={}, systemId={}) - start", publicId, systemId);

        // No DTD metadata wanted/available
        if (_dtdHandler == null || !_dtdHandler.isDtdPresent())
        {
            return new InputSource(new StringReader(""));
        }
        return null;
    }

    ////////////////////////////////////////////////////////////////////////////
    // ErrorHandler interface

    public void error(SAXParseException e) throws SAXException
    {
        throw e;

    }

    ////////////////////////////////////////////////////////////////////////
    // ContentHandler interface

    public void startElement(String uri, String localName, String qName,
            Attributes attributes) throws SAXException
    {
        if (logger.isDebugEnabled())
            logger.debug("startElement(uri={}, localName={}, qName={}, attributes={}) - start",
                    new Object[] { uri, localName, qName, attributes });

        try
        {
            ITableMetaData activeMetaData = getActiveMetaData();
            // Start of dataset
            if (activeMetaData == null && qName.equals(DATASET))
            {
                _consumer.startDataSet();
                _orderedTableNameMap = new OrderedTableNameMap(_caseSensitiveTableNames);
                return;
            }

            // New table
            if (isNewTable(qName))
            {
                // If not first table, notify end of previous table to consumer
                if (activeMetaData != null)
                {
                    _consumer.endTable();
                }

                // In FlatXML the table might have appeared before already, so check for this
                if(_orderedTableNameMap.containsTable(qName))
                {
                    activeMetaData = (ITableMetaData)_orderedTableNameMap.get(qName);
                    _orderedTableNameMap.setLastTable(qName);
                }
                else
                {
                    activeMetaData = createTableMetaData(qName, attributes);
                    _orderedTableNameMap.add(activeMetaData.getTableName(), activeMetaData);
                }
                
                // Notify start of new table to consumer
                _consumer.startTable(activeMetaData);
                _lineNumber = 0;
            }

            // Row notification
            int attributesLength = attributes.getLength();
            if (attributesLength > 0)
            {
                // If we do not have a DTD
                if (_dtdHandler == null || !_dtdHandler.isDtdPresent())
                {
                    handleMissingColumns(attributes);
                    // Since a new MetaData object was created assign it to the local variable
                    activeMetaData = getActiveMetaData();
                }

                _lineNumber++;
                _lineNumberGlobal++;
                Column[] columns = activeMetaData.getColumns();
                Object[] rowValues = new Object[columns.length];
                for (int i = 0; i < attributesLength; i++)
                {
                    determineAndSetRowValue(attributes, activeMetaData,
                            rowValues, i);
                }
                _consumer.row(rowValues);
            }
        }
        catch (DataSetException e)
        {
            throw new SAXException(e);
        }
    }

    protected void determineAndSetRowValue(Attributes attributes,
            ITableMetaData activeMetaData, Object[] rowValues, int i)
            throws DataSetException, NoSuchColumnException
    {
        String attributeQName = attributes.getQName(i);
        String attributeValue = attributes.getValue(i);
        try
        {
            int colIndex = activeMetaData.getColumnIndex(attributeQName);
            rowValues[colIndex] = attributeValue;
        }
        catch (NoSuchColumnException e)
        {
            // since missing columns have already been handled above, we will only need
            // to care about NoSuchColumnExceptions if something's gone wrong and we're
            // looking for a nonexistant column that should have been sensed
            if (_columnSensing)
            {
                throw e;
            }
        }
    }

    public void endElement(String uri, String localName, String qName) throws SAXException
    {
        if (logger.isDebugEnabled())
            logger.debug("endElement(uri={}, localName={}, qName={}) - start",
                    new Object[]{ uri, localName, qName });

        // End of dataset
        if (qName.equals(DATASET))
        {
            try
            {
                // Notify end of active table to consumer
                if (getActiveMetaData() != null)
                {
                    _consumer.endTable();
                }

                // Notify end of dataset to consumer
                _consumer.endDataSet();
            }
            catch (DataSetException e)
            {
                throw new SAXException(e);
            }
        }
    }

    private static class FlatDtdHandler extends FlatDtdProducer
    {
        /**
         * Logger for this class
         */
        private final Logger logger = LoggerFactory.getLogger(FlatDtdHandler.class);

        private boolean _dtdPresent = false;
        private FlatXmlProducer xmlProducer;

        public FlatDtdHandler(FlatXmlProducer xmlProducer)
        {
            this.xmlProducer = xmlProducer;
        }

        public boolean isDtdPresent() 
        {
            return _dtdPresent;
        }
        
        ////////////////////////////////////////////////////////////////////////////
        // LexicalHandler interface

        public void startDTD(String name, String publicId, String systemId)
                throws SAXException
        {
            if (logger.isDebugEnabled())
                logger.debug("startDTD(name={}, publicId={}, systemId={}) - start",
                        new Object[] { name, publicId, systemId });

            _dtdPresent = true;
            try
            {
                // Cache the DTD content to use it as metadata
                FlatDtdDataSet metaDataSet = new FlatDtdDataSet();
                this.setConsumer(metaDataSet);
                // Set the metaData on the xmlProducer
                xmlProducer._metaDataSet = metaDataSet;

                super.startDTD(name, publicId, systemId);
            }
            catch (DataSetException e)
            {
                throw new SAXException(e);
            }
        }
    }
}