/*
    * Copyright 2007 Kasper B. Graversen
    * 
    * Licensed under the Apache License, Version 2.0 (the "License");
    * you may not use this file except in compliance with the License.
    * You may obtain a copy of the License at
    * 
    *     http://www.apache.org/licenses/LICENSE-2.0
    * 
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.supercsv.io;
  
  import java.io.IOException;
  import java.io.Reader;
  import java.util.List;
  
  import org.supercsv.comment.CommentMatcher;
  import org.supercsv.exception.SuperCsvException;
  import org.supercsv.prefs.CsvPreference;
  
  /**
   * Reads the CSV file, line by line. If you want the line-reading functionality of this class, but want to define your
   * own implementation of {@link #readColumns(List)}, then consider writing your own Tokenizer by extending
   * AbstractTokenizer.
   * 
   * @author Kasper B. Graversen
   * @author James Bassett
   */
  public class Tokenizer extends AbstractTokenizer {
  	
  	private static final char NEWLINE = '\n';
  	
  	private static final char SPACE = ' ';
  	
  	private final StringBuilder currentColumn = new StringBuilder();
  	
  	/* the raw, untokenized CSV row (may span multiple lines) */
  	private final StringBuilder currentRow = new StringBuilder();
  	
  	private final int quoteChar;
  	
  	private final int delimeterChar;
  	
  	private final boolean surroundingSpacesNeedQuotes;
  	
  	private final boolean ignoreEmptyLines;
  	
  	private final CommentMatcher commentMatcher;
  
  	private final int maxLinesPerRow;
  	
  	/**
  	 * Enumeration of tokenizer states. QUOTE_MODE is activated between quotes.
  	 */
  	private enum TokenizerState {
  		NORMAL, QUOTE_MODE;
  	}
  	
  	/**
  	 * Constructs a new <tt>Tokenizer</tt>, which reads the CSV file, line by line.
  	 * 
  	 * @param reader
  	 *            the reader
  	 * @param preferences
  	 *            the CSV preferences
  	 * @throws NullPointerException
  	 *             if reader or preferences is null
  	 */
  	public Tokenizer(final Reader reader, final CsvPreference preferences) {
  		super(reader, preferences);
  		this.quoteChar = preferences.getQuoteChar();
  		this.delimeterChar = preferences.getDelimiterChar();
  		this.surroundingSpacesNeedQuotes = preferences.isSurroundingSpacesNeedQuotes();
  		this.ignoreEmptyLines = preferences.isIgnoreEmptyLines();
  		this.commentMatcher = preferences.getCommentMatcher();
  		this.maxLinesPerRow = preferences.getMaxLinesPerRow();
  	}
  	
  	/**
  	 * {@inheritDoc}
  	 */
  	public boolean readColumns(final List<String> columns) throws IOException {
  		
  		if( columns == null ) {
  			throw new NullPointerException("columns should not be null");
  		}
  		
  		// clear the reusable List and StringBuilders
  		columns.clear();
  		currentColumn.setLength(0);
  		currentRow.setLength(0);
  		
  		// read a line (ignoring empty lines/comments if necessary)
  		String line;
 		do {
 			line = readLine();
 			if( line == null ) {
 				return false; // EOF
 			}
 		}
 		while( ignoreEmptyLines && line.length() == 0 || (commentMatcher != null && commentMatcher.isComment(line)) );
 		
 		// update the untokenized CSV row
 		currentRow.append(line);
 		
 		// process each character in the line, catering for surrounding quotes (QUOTE_MODE)
 		TokenizerState state = TokenizerState.NORMAL;
 		int quoteScopeStartingLine = -1; // the line number where a potential multi-line cell starts
 		int potentialSpaces = 0; // keep track of spaces (so leading/trailing space can be removed if required)
 		int charIndex = 0;
 		while( true ) {
 			boolean endOfLineReached = charIndex == line.length();
 			
 			if( endOfLineReached )
 			{
 				if( TokenizerState.NORMAL.equals(state) ) {
 					/*
 					 * Newline. Add any required spaces (if surrounding spaces don't need quotes) and return (we've read
 					 * a line!).
 					 */
 					if( !surroundingSpacesNeedQuotes ) {
 						appendSpaces(currentColumn, potentialSpaces);
 					}
 					columns.add(currentColumn.length() > 0 ? currentColumn.toString() : null); // "" -> null
 					return true;
 				}
 				else
 				{
 					/*
 					 * Newline. Doesn't count as newline while in QUOTESCOPE. Add the newline char, reset the charIndex
 					 * (will update to 0 for next iteration), read in the next line, then then continue to next
 					 * character.
 					 */
 					currentColumn.append(NEWLINE);
 					currentRow.append(NEWLINE); // specific line terminator lost, \n will have to suffice
 					
 					charIndex = 0;
 
 					if (maxLinesPerRow > 0 && getLineNumber() - quoteScopeStartingLine + 1 >= maxLinesPerRow) {
 						/*
 						 * The quoted section that is being parsed spans too many lines, so to avoid excessive memory
 						 * usage parsing something that is probably human error anyways, throw an exception. If each
 						 * row is suppose to be a single line and this has been exceeded, throw a more descriptive
 						 * exception
 						 */
 						String msg = maxLinesPerRow == 1 ?
 								String.format("unexpected end of line while reading quoted column on line %d",
 											  getLineNumber()) :
 								String.format("max number of lines to read exceeded while reading quoted column" +
 											  " beginning on line %d and ending on line %d",
 											  quoteScopeStartingLine, getLineNumber());
 						throw new SuperCsvException(msg);
 					}
 					else if( (line = readLine()) == null ) {
 						throw new SuperCsvException(
 							String
 								.format(
 									"unexpected end of file while reading quoted column beginning on line %d and ending on line %d",
 									quoteScopeStartingLine, getLineNumber()));
 					}
 					
 					currentRow.append(line); // update untokenized CSV row
 					
 				    if (line.length() == 0){
 				    	// consecutive newlines
                         continue;
 				    }
 				}
 			}
 			
 			final char c = line.charAt(charIndex);
 			
 			if( TokenizerState.NORMAL.equals(state) ) {
 				
 				/*
 				 * NORMAL mode (not within quotes).
 				 */
 				
 				if( c == delimeterChar ) {
 					/*
 					 * Delimiter. Save the column (trim trailing space if required) then continue to next character.
 					 */
 					if( !surroundingSpacesNeedQuotes ) {
 						appendSpaces(currentColumn, potentialSpaces);
 					}
 					columns.add(currentColumn.length() > 0 ? currentColumn.toString() : null); // "" -> null
 					potentialSpaces = 0;
 					currentColumn.setLength(0);
 					
 				} else if( c == SPACE ) {
 					/*
 					 * Space. Remember it, then continue to next character.
 					 */
 					potentialSpaces++;
 					
 				}
 				else if( c == quoteChar ) {
 					/*
 					 * A single quote ("). Update to QUOTESCOPE (but don't save quote), then continue to next character.
 					 */
 					state = TokenizerState.QUOTE_MODE;
 					quoteScopeStartingLine = getLineNumber();
 					
 					// cater for spaces before a quoted section (be lenient!)
 					if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) {
 						appendSpaces(currentColumn, potentialSpaces);
 					}
 					potentialSpaces = 0;
 					
 				} else {
 					/*
 					 * Just a normal character. Add any required spaces (but trim any leading spaces if surrounding
 					 * spaces need quotes), add the character, then continue to next character.
 					 */
 					if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) {
 						appendSpaces(currentColumn, potentialSpaces);
 					}
 					
 					potentialSpaces = 0;
 					currentColumn.append(c);
 				}
 				
 			} else {
 				
 				/*
 				 * QUOTE_MODE (within quotes).
 				 */
 				
 				if( c == quoteChar ) {
 					int nextCharIndex = charIndex + 1;
 					boolean availableCharacters = nextCharIndex < line.length();
 					boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar;
 					if( nextCharIsQuote ) {
 						/*
 						 * An escaped quote (""). Add a single quote, then move the cursor so the next iteration of the
 						 * loop will read the character following the escaped quote.
 						 */
 						currentColumn.append(c);
 						charIndex++;
 						
 					} else {
 						/*
 						 * A single quote ("). Update to NORMAL (but don't save quote), then continue to next character.
 						 */
 						state = TokenizerState.NORMAL;
 						quoteScopeStartingLine = -1; // reset ready for next multi-line cell
 					}
 				} else {
 					/*
 					 * Just a normal character, delimiter (they don't count in QUOTESCOPE) or space. Add the character,
 					 * then continue to next character.
 					 */
 					currentColumn.append(c);
 				}
 			}
 			
 			charIndex++; // read next char of the line
 		}
 	}
 	
 	/**
 	 * Appends the required number of spaces to the StringBuilder.
 	 * 
 	 * @param sb
 	 *            the StringBuilder
 	 * @param spaces
 	 *            the required number of spaces to append
 	 */
 	private static void appendSpaces(final StringBuilder sb, final int spaces) {
 		for( int i = 0; i < spaces; i++ ) {
 			sb.append(SPACE);
 		}
 	}
 	
 	/**
 	 * {@inheritDoc}
 	 */
 	public String getUntokenizedRow() {
 		return currentRow.toString();
 	}
 }