nlsr_tokenizer.cpp

#include <iostream>
#include <boost/tokenizer.hpp>
#include <boost/algorithm/string.hpp>
#include <string>
#include <algorithm>

#include "nlsr_tokenizer.hpp"

using namespace std;
using namespace boost;

void 
nlsrTokenizer::makeToken(){
	char_separator<char> sep(seps.c_str());
	tokenizer< char_separator<char> >tokens(originalString, sep);
	tokenizer< char_separator<char> >::iterator tok_iter = tokens.begin();
	
	string ft(*tok_iter);
	firstToken=ft;
	++tok_iter;

	for ( ;tok_iter != tokens.end(); ++tok_iter){
		string oneToken(*tok_iter);
		this->insertToken(oneToken);
		restOfTheLine+=oneToken;
		restOfTheLine+=seps;
  	}

	trim(restOfTheLine);
}

void 
nlsrTokenizer::insertToken(const string& token){
	tokenList.push_back(token);
}

int 
nlsrTokenizer::getTokenPosition(string& token){
	int pos=-1;
	int i=1;

	for(std::list<string>::iterator it=tokenList.begin();it!=tokenList.end();it++){
		if( (*it) == token ){
			break;
		}
		i++;
	}

	if( i < tokenList.size() ){
		pos=i;
	}

	return pos;
}

string 
nlsrTokenizer::getTokenString(int from , int to){
	string returnString;
	if ( from >=0 && to < tokenList.size()){
		int i=0;
		for(std::list<string>::iterator it=tokenList.begin();
													it!=tokenList.end();it++){
			i++;
			if( i >= from && i<= to ){
				string oneToken((*it));
				returnString+=seps;
				returnString+=oneToken;
				
			}
			
		}
	}

	trim(returnString);
	return returnString;
}

string 
nlsrTokenizer::getTokenString(int from){
	string returnString;
	if ( from >=0 && from < tokenList.size()){
		int i=0;
		for(std::list<string>::iterator it=tokenList.begin();
													it!=tokenList.end();it++){
			i++;
			if( i >= from){
				string oneToken((*it));
				returnString+=seps;
				returnString+=oneToken;
				
			}
			
		}
	}

	trim(returnString);
	return returnString;
}

static bool
tokenCompare(string& s1, string& s2){
	return s1==s2;
}

bool
nlsrTokenizer::doesTokenExist(string token){
	std::list<string >::iterator it = std::find_if( tokenList.begin(), 
								tokenList.end(),	
   								bind(&tokenCompare, _1 , token));

	if( it != tokenList.end() ){
		return true;
	}

	return false;
}