diff --git a/src/nlsr_tokenizer.cpp b/src/nlsr_tokenizer.cpp
new file mode 100644
index 0000000..e03d2d2
--- /dev/null
+++ b/src/nlsr_tokenizer.cpp
@@ -0,0 +1,116 @@
+#include <iostream>
+#include <boost/tokenizer.hpp>
+#include <boost/algorithm/string.hpp>
+#include <string>
+#include <algorithm>
+
+#include "nlsr_tokenizer.hpp"
+
+using namespace std;
+using namespace boost;
+
+void 
+nlsrTokenizer::makeToken(){
+	char_separator<char> sep(seps.c_str());
+	tokenizer< char_separator<char> >tokens(originalString, sep);
+	tokenizer< char_separator<char> >::iterator tok_iter = tokens.begin();
+	
+	string ft(*tok_iter);
+	firstToken=ft;
+	++tok_iter;
+
+	for ( ;tok_iter != tokens.end(); ++tok_iter){
+		string oneToken(*tok_iter);
+		this->insertToken(oneToken);
+		restOfTheLine+=oneToken;
+		restOfTheLine+=seps;
+  	}
+
+	trim(restOfTheLine);
+}
+
+void 
+nlsrTokenizer::insertToken(const string& token){
+	tokenList.push_back(token);
+}
+
+int 
+nlsrTokenizer::getTokenPosition(string& token){
+	int pos=-1;
+	int i=1;
+
+	for(std::list<string>::iterator it=tokenList.begin();it!=tokenList.end();it++){
+		if( (*it) == token ){
+			break;
+		}
+		i++;
+	}
+
+	if( i < tokenList.size() ){
+		pos=i;
+	}
+
+	return pos;
+}
+
+string 
+nlsrTokenizer::getTokenString(int from , int to){
+	string returnString;
+	if ( from >=0 && to < tokenList.size()){
+		int i=0;
+		for(std::list<string>::iterator it=tokenList.begin();
+													it!=tokenList.end();it++){
+			i++;
+			if( i >= from && i<= to ){
+				string oneToken((*it));
+				returnString+=seps;
+				returnString+=oneToken;
+				
+			}
+			
+		}
+	}
+
+	trim(returnString);
+	return returnString;
+}
+
+string 
+nlsrTokenizer::getTokenString(int from){
+	string returnString;
+	if ( from >=0 && from < tokenList.size()){
+		int i=0;
+		for(std::list<string>::iterator it=tokenList.begin();
+													it!=tokenList.end();it++){
+			i++;
+			if( i >= from){
+				string oneToken((*it));
+				returnString+=seps;
+				returnString+=oneToken;
+				
+			}
+			
+		}
+	}
+
+	trim(returnString);
+	return returnString;
+}
+
+static bool
+tokenCompare(string& s1, string& s2){
+	return s1==s2;
+}
+
+bool
+nlsrTokenizer::doesTokenExist(string token){
+	std::list<string >::iterator it = std::find_if( tokenList.begin(), 
+								tokenList.end(),	
+   								bind(&tokenCompare, _1 , token));
+
+	if( it != tokenList.end() ){
+		return true;
+	}
+
+	return false;
+}