View Javadoc
1   /*
2    *
3    * The DbUnit Database Testing Framework
4    * Copyright (C)2002-2004, DbUnit.org
5    *
6    * This library is free software; you can redistribute it and/or
7    * modify it under the terms of the GNU Lesser General Public
8    * License as published by the Free Software Foundation; either
9    * version 2.1 of the License, or (at your option) any later version.
10   *
11   * This library is distributed in the hope that it will be useful,
12   * but WITHOUT ANY WARRANTY; without even the implied warranty of
13   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14   * Lesser General Public License for more details.
15   *
16   * You should have received a copy of the GNU Lesser General Public
17   * License along with this library; if not, write to the Free Software
18   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19   *
20   */
21  package org.dbunit.dataset.xml;
22  
23  import java.io.IOException;
24  import java.io.StringReader;
25  import java.util.ArrayList;
26  import java.util.Iterator;
27  import java.util.List;
28  
29  import javax.xml.parsers.ParserConfigurationException;
30  import javax.xml.parsers.SAXParserFactory;
31  
32  import org.dbunit.dataset.Column;
33  import org.dbunit.dataset.DataSetException;
34  import org.dbunit.dataset.DefaultTableMetaData;
35  import org.dbunit.dataset.IDataSet;
36  import org.dbunit.dataset.ITableMetaData;
37  import org.dbunit.dataset.NoSuchColumnException;
38  import org.dbunit.dataset.OrderedTableNameMap;
39  import org.dbunit.dataset.datatype.DataType;
40  import org.dbunit.dataset.stream.BufferedConsumer;
41  import org.dbunit.dataset.stream.DefaultConsumer;
42  import org.dbunit.dataset.stream.IDataSetConsumer;
43  import org.dbunit.dataset.stream.IDataSetProducer;
44  import org.slf4j.Logger;
45  import org.slf4j.LoggerFactory;
46  import org.xml.sax.Attributes;
47  import org.xml.sax.ContentHandler;
48  import org.xml.sax.EntityResolver;
49  import org.xml.sax.InputSource;
50  import org.xml.sax.SAXException;
51  import org.xml.sax.SAXParseException;
52  import org.xml.sax.XMLReader;
53  import org.xml.sax.helpers.DefaultHandler;
54  
55  /**
56   * @author Manuel Laflamme
57   * @author Last changed by: $Author$
58   * @version $Revision$ $Date$
59   * @since 1.5 (Apr 18, 2003)
60   */
61  public class FlatXmlProducer extends DefaultHandler implements IDataSetProducer, ContentHandler
62  {
63  
64      /**
65       * Logger for this class
66       */
67      private static final Logger logger = LoggerFactory.getLogger(FlatXmlProducer.class);
68  
69      private static final IDataSetConsumer EMPTY_CONSUMER = new DefaultConsumer();
70      private static final String DATASET = "dataset";
71  
72      private final InputSource _inputSource;
73      private final EntityResolver _resolver;
74      private boolean _validating = false;
75      /**
76       * The dataset used to retrieve the metadata for the tables via {@link IDataSet#getTableMetaData(String)}. 
77       * Can be null
78       */
79      private IDataSet _metaDataSet;
80      /**
81       * The DTD handler which is used to parse a DTD if available. The result of the parsing is stored in
82       * {@link #_metaDataSet}.
83       */
84      private FlatDtdHandler _dtdHandler;
85      
86      /**
87       * The current line number in the current table
88       */
89      private int _lineNumber = 0;
90      /**
91       * The current line number
92       */
93      private int _lineNumberGlobal = 0;
94      /**
95       * Whether the column sensing feature should be used to dynamically recognize new columns
96       * during the parse process.
97       */
98      private boolean _columnSensing = false;
99      private boolean _caseSensitiveTableNames;
100 
101     /**
102      * The consumer which is responsible for creating the datasets and tables
103      */
104     private IDataSetConsumer _consumer = EMPTY_CONSUMER;
105     /**
106      * The ordered table name map which also holds the currently active {@link ITableMetaData}
107      */
108     private OrderedTableNameMap _orderedTableNameMap;
109 
110     
111     public FlatXmlProducer(InputSource xmlSource)
112     {
113         this(xmlSource, true);
114     }
115 
116     public FlatXmlProducer(InputSource xmlSource, boolean dtdMetadata)
117     {
118         this(xmlSource, dtdMetadata, false);
119     }
120 
121     public FlatXmlProducer(InputSource xmlSource, IDataSet metaDataSet)
122     {
123         _inputSource = xmlSource;
124         _metaDataSet = metaDataSet;
125         _resolver = this;
126         _caseSensitiveTableNames = metaDataSet.isCaseSensitiveTableNames();
127         initialize(false);
128     }
129 
130     public FlatXmlProducer(InputSource xmlSource, EntityResolver resolver)
131     {
132         _inputSource = xmlSource;
133         _resolver = resolver;
134         initialize(true);
135     }
136     
137     /**
138      * @param xmlSource The input datasource
139      * @param dtdMetadata Whether or not DTD metadata is available to parse via a DTD handler
140      * @param columnSensing Whether or not the column sensing feature should be used (see FAQ)
141      */
142     public FlatXmlProducer(InputSource xmlSource, boolean dtdMetadata, boolean columnSensing)
143     {
144         this(xmlSource, dtdMetadata, columnSensing, false);
145     }
146     
147     /**
148      * @param xmlSource The input datasource
149      * @param dtdMetadata Whether or not DTD metadata is available to parse via a DTD handler
150      * @param columnSensing Whether or not the column sensing feature should be used (see FAQ)
151      * @param caseSensitiveTableNames Whether or not this dataset should use case sensitive table names
152      * @since 2.4.2
153      */
154     public FlatXmlProducer(InputSource xmlSource, boolean dtdMetadata, boolean columnSensing, boolean caseSensitiveTableNames)
155     {
156         _inputSource = xmlSource;
157         _columnSensing = columnSensing;
158         _caseSensitiveTableNames = caseSensitiveTableNames;
159         _resolver = this;
160         initialize(dtdMetadata);
161     }
162 
163 
164     
165     private void initialize(boolean dtdMetadata)
166     {
167         if (dtdMetadata)
168         {
169             this._dtdHandler = new FlatDtdHandler(this);
170         }
171     }
172     
173     /**
174      * @return Whether or not this producer works case sensitively
175      * @since 2.4.7
176      */
177     public boolean isCaseSensitiveTableNames() {
178         return _caseSensitiveTableNames;
179     }
180 
181     private ITableMetaData createTableMetaData(String tableName, Attributes attributes) throws DataSetException
182     {
183     	if (logger.isDebugEnabled())
184     		logger.debug("createTableMetaData(tableName={}, attributes={}) - start", tableName, attributes);
185 
186     	// First try to find it in the DTD's dataset
187         if (_metaDataSet != null)
188         {
189             return _metaDataSet.getTableMetaData(tableName);
190         }
191 
192         // Create metadata from attributes
193         Column[] columns = new Column[attributes.getLength()];
194         for (int i = 0; i < attributes.getLength(); i++)
195         {
196             columns[i] = new Column(attributes.getQName(i), DataType.UNKNOWN);
197         }
198 
199         return new DefaultTableMetaData(tableName, columns);
200     }
201     
202     
203     /**
204      * merges the existing columns with the potentially new ones.
205      * @param columnsToMerge List of extra columns found, which need to be merge back into the metadata.
206      * @return ITableMetaData The merged metadata object containing the new columns
207      * @throws DataSetException
208      */
209     private ITableMetaData mergeTableMetaData(List columnsToMerge, ITableMetaData originalMetaData) throws DataSetException
210     {
211         Column[] columns = new Column[originalMetaData.getColumns().length + columnsToMerge.size()];
212         System.arraycopy(originalMetaData.getColumns(), 0, columns, 0, originalMetaData.getColumns().length);
213         
214         for (int i = 0; i < columnsToMerge.size(); i++)
215         {
216         	Column column = (Column)columnsToMerge.get(i);
217         	columns[columns.length - columnsToMerge.size() + i] = column;
218         }
219     	
220     	return new DefaultTableMetaData(originalMetaData.getTableName(), columns);
221     }
222     
223     /**
224      * @return The currently active table metadata or <code>null</code> if no active
225      * metadata exists.
226      */
227     private ITableMetaData getActiveMetaData()
228     {
229         if(_orderedTableNameMap != null)
230         {
231             String lastTableName = _orderedTableNameMap.getLastTableName();
232             if(lastTableName != null)
233             {
234                 return (ITableMetaData) _orderedTableNameMap.get(lastTableName);
235             }
236             else
237             {
238                 return null;
239             }
240         }
241         else
242         {
243             return null;
244         }
245         
246     }
247     
248     /**
249      * @param tableName
250      * @return <code>true</code> if the given tableName is a new one
251      * which means that it differs from the last active table name.
252      */
253     private boolean isNewTable(String tableName) 
254     {
255         return !_orderedTableNameMap.isLastTable(tableName);
256     }
257 
258 
259     /**
260      * parses the attributes in the current row, and checks whether a new column 
261      * is found.
262      * 
263      * <p>Depending on the value of the <code>columnSensing</code> flag, the appropriate
264      * action is taken:</p>
265      * 
266      * <ul>
267      *   <li>If it is true, the new column is merged back into the metadata;</li>
268      *   <li>If not, a warning message is displayed.</li>
269      * </ul>
270      * 
271      * @param attributes  Attributed for the current row.
272      * @throws DataSetException
273      */
274 	protected void handleMissingColumns(Attributes attributes)
275 			throws DataSetException
276 	{
277 		List columnsToMerge = new ArrayList();
278 		
279 		ITableMetaData activeMetaData = getActiveMetaData();
280 		// Search all columns that do not yet exist and collect them
281 		int attributeLength = attributes.getLength();
282 		for (int i = 0 ; i < attributeLength; i++)
283 		{
284 			try {
285 			    activeMetaData.getColumnIndex(attributes.getQName(i));
286 			} 
287 			catch (NoSuchColumnException e) {
288 				columnsToMerge.add(new Column(attributes.getQName(i), DataType.UNKNOWN));
289 			}
290 		}
291 		
292 		if (!columnsToMerge.isEmpty())
293 		{
294 			if (_columnSensing)
295 	    	{
296 	    		logger.debug("Column sensing enabled. Will create a new metaData with potentially new columns if needed");
297 	    		activeMetaData = mergeTableMetaData(columnsToMerge, activeMetaData);
298 	    		_orderedTableNameMap.update(activeMetaData.getTableName(), activeMetaData);
299 	    		// We also need to recreate the table, copying the data already collected from the old one to the new one
300 	    		_consumer.startTable(activeMetaData);
301 	    	} 
302 	    	else
303 	    	{
304 	    		StringBuffer extraColumnNames = new StringBuffer();
305 	    		for (Iterator i = columnsToMerge.iterator(); i.hasNext();) {
306 					Column col = (Column) i.next();
307 					extraColumnNames.append(extraColumnNames.length() > 0 ? "," : "").append(col.getColumnName());
308 				}
309 	    	    String msg = "Extra columns (" + extraColumnNames.toString() + ") on line " + (_lineNumber + 1)
310 						+ " for table " + activeMetaData.getTableName() + " (global line number is "
311 						+ _lineNumberGlobal + "). Those columns will be ignored.";
312 	    	    msg += "\n\tPlease add the extra columns to line 1,"
313                     + " or use a DTD to make sure the value of those columns are populated" 
314                     + " or specify 'columnSensing=true' for your FlatXmlProducer.";
315 	    	    msg += "\n\tSee FAQ for more details.";
316 	    		logger.warn(msg);
317 	    	}
318 		}
319 	}
320 	
321 	public void setColumnSensing(boolean columnSensing)
322 	{
323 		_columnSensing = columnSensing;
324 	}
325 
326     public void setValidating(boolean validating)
327     {
328         _validating = validating;
329     }
330 
331     ////////////////////////////////////////////////////////////////////////////
332     // IDataSetProducer interface
333 
334     public void setConsumer(IDataSetConsumer consumer) throws DataSetException
335     {
336         logger.debug("setConsumer(consumer) - start");
337 
338         if(this._columnSensing) {
339             _consumer = new BufferedConsumer(consumer);
340         }
341         else {
342             _consumer = consumer;
343         }
344     }
345 
346     public void produce() throws DataSetException
347     {
348         logger.debug("produce() - start");
349 
350         try
351         {
352             SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
353             saxParserFactory.setValidating(_validating);
354             XMLReader xmlReader = saxParserFactory.newSAXParser().getXMLReader();
355 
356             if(_dtdHandler != null)
357             {
358                 FlatDtdHandler.setLexicalHandler(xmlReader, _dtdHandler);
359                 FlatDtdHandler.setDeclHandler(xmlReader, _dtdHandler);
360             }
361 
362             xmlReader.setContentHandler(this);
363             xmlReader.setErrorHandler(this);
364             xmlReader.setEntityResolver(_resolver);
365             xmlReader.parse(_inputSource);
366         }
367         catch (ParserConfigurationException e)
368         {
369             throw new DataSetException(e);
370         }
371         catch (SAXException e)
372         {
373             DataSetException exceptionToRethrow = XmlProducer.buildException(e);
374             throw exceptionToRethrow;
375         }
376         catch (IOException e)
377         {
378             throw new DataSetException(e);
379         }
380     }
381 
382     ////////////////////////////////////////////////////////////////////////////
383     // EntityResolver interface
384 
385     public InputSource resolveEntity(String publicId, String systemId)
386             throws SAXException
387     {
388         logger.debug("resolveEntity(publicId={}, systemId={}) - start", publicId, systemId);
389 
390         // No DTD metadata wanted/available
391         if (_dtdHandler == null || !_dtdHandler.isDtdPresent())
392         {
393             return new InputSource(new StringReader(""));
394         }
395         return null;
396     }
397 
398     ////////////////////////////////////////////////////////////////////////////
399     // ErrorHandler interface
400 
401     public void error(SAXParseException e) throws SAXException
402     {
403         throw e;
404 
405     }
406 
407     ////////////////////////////////////////////////////////////////////////
408     // ContentHandler interface
409 
410     public void startElement(String uri, String localName, String qName,
411             Attributes attributes) throws SAXException
412     {
413     	if (logger.isDebugEnabled())
414     		logger.debug("startElement(uri={}, localName={}, qName={}, attributes={}) - start",
415     				new Object[] { uri, localName, qName, attributes });
416 
417         try
418         {
419             ITableMetaData activeMetaData = getActiveMetaData();
420             // Start of dataset
421             if (activeMetaData == null && qName.equals(DATASET))
422             {
423                 _consumer.startDataSet();
424                 _orderedTableNameMap = new OrderedTableNameMap(_caseSensitiveTableNames);
425                 return;
426             }
427 
428             // New table
429             if (isNewTable(qName))
430             {
431                 // If not first table, notify end of previous table to consumer
432                 if (activeMetaData != null)
433                 {
434                     _consumer.endTable();
435                 }
436 
437                 // In FlatXML the table might have appeared before already, so check for this
438                 if(_orderedTableNameMap.containsTable(qName))
439                 {
440                     activeMetaData = (ITableMetaData)_orderedTableNameMap.get(qName);
441                     _orderedTableNameMap.setLastTable(qName);
442                 }
443                 else
444                 {
445                     activeMetaData = createTableMetaData(qName, attributes);
446                     _orderedTableNameMap.add(activeMetaData.getTableName(), activeMetaData);
447                 }
448                 
449                 // Notify start of new table to consumer
450                 _consumer.startTable(activeMetaData);
451                 _lineNumber = 0;
452             }
453 
454             // Row notification
455             if (attributes.getLength() > 0)
456             {
457                 // If we do not have a DTD
458             	if (_dtdHandler == null || !_dtdHandler.isDtdPresent())
459             	{
460 	            	handleMissingColumns(attributes);
461 	            	// Since a new MetaData object was created assign it to the local variable
462 	            	activeMetaData = getActiveMetaData();
463             	}
464             	
465             	_lineNumber++;
466             	_lineNumberGlobal++;
467                 Column[] columns = activeMetaData.getColumns();
468                 Object[] rowValues = new Object[columns.length];
469                 for (int i = 0; i < columns.length; i++)
470                 {
471                     Column column = columns[i];
472                     rowValues[i] = attributes.getValue(column.getColumnName());
473                 }
474                 _consumer.row(rowValues);
475             }
476         }
477         catch (DataSetException e)
478         {
479             throw new SAXException(e);
480         }
481     }
482 
483     public void endElement(String uri, String localName, String qName) throws SAXException
484     {
485     	if (logger.isDebugEnabled())
486     		logger.debug("endElement(uri={}, localName={}, qName={}) - start",
487     				new Object[]{ uri, localName, qName });
488 
489         // End of dataset
490         if (qName.equals(DATASET))
491         {
492             try
493             {
494                 // Notify end of active table to consumer
495                 if (getActiveMetaData() != null)
496                 {
497                     _consumer.endTable();
498                 }
499 
500                 // Notify end of dataset to consumer
501                 _consumer.endDataSet();
502             }
503             catch (DataSetException e)
504             {
505                 throw new SAXException(e);
506             }
507         }
508     }
509 
510     private static class FlatDtdHandler extends FlatDtdProducer
511     {
512         /**
513          * Logger for this class
514          */
515         private final Logger logger = LoggerFactory.getLogger(FlatDtdHandler.class);
516 
517         private boolean _dtdPresent = false;
518         private FlatXmlProducer xmlProducer;
519 
520         public FlatDtdHandler(FlatXmlProducer xmlProducer)
521         {
522             this.xmlProducer = xmlProducer;
523         }
524 
525         public boolean isDtdPresent() 
526         {
527             return _dtdPresent;
528         }
529         
530         ////////////////////////////////////////////////////////////////////////////
531         // LexicalHandler interface
532 
533         public void startDTD(String name, String publicId, String systemId)
534                 throws SAXException
535         {
536         	if (logger.isDebugEnabled())
537         		logger.debug("startDTD(name={}, publicId={}, systemId={}) - start",
538         				new Object[] { name, publicId, systemId });
539 
540             _dtdPresent = true;
541             try
542             {
543                 // Cache the DTD content to use it as metadata
544                 FlatDtdDataSet metaDataSet = new FlatDtdDataSet();
545                 this.setConsumer(metaDataSet);
546                 // Set the metaData on the xmlProducer
547                 xmlProducer._metaDataSet = metaDataSet;
548 
549                 super.startDTD(name, publicId, systemId);
550             }
551             catch (DataSetException e)
552             {
553                 throw new SAXException(e);
554             }
555         }
556     }
557 }