First commit:CPP
diff --git a/nlsr_tokenizer.cpp b/nlsr_tokenizer.cpp
new file mode 100644
index 0000000..7c82ecd
--- /dev/null
+++ b/nlsr_tokenizer.cpp
@@ -0,0 +1,114 @@
+#include <iostream>
+#include <boost/tokenizer.hpp>
+#include <boost/algorithm/string.hpp>
+#include <string>
+#include <algorithm>
+
+#include "nlsr_tokenizer.hpp"
+
+using namespace std;
+using namespace boost;
+
+void
+nlsrTokenizer::makeToken(){
+ char_separator<char> sep(seps.c_str());
+ tokenizer< char_separator<char> >tokens(originalString, sep);
+ tokenizer< char_separator<char> >::iterator tok_iter = tokens.begin();
+
+ string ft(*tok_iter);
+ firstToken=ft;
+ ++tok_iter;
+
+ for ( ;tok_iter != tokens.end(); ++tok_iter){
+ string oneToken(*tok_iter);
+ this->insertToken(oneToken);
+ restOfTheLine+=oneToken;
+ restOfTheLine+=seps;
+ }
+
+ trim(restOfTheLine);
+}
+
+void
+nlsrTokenizer::insertToken(const string& token){
+ tokenList.push_back(token);
+}
+
+int
+nlsrTokenizer::getTokenPosition(string& token){
+ int pos=-1;
+ int i=1;
+
+ for(std::list<string>::iterator it=tokenList.begin();it!=tokenList.end();it++){
+ if( (*it) == token ){
+ break;
+ }
+ i++;
+ }
+
+ if( i < tokenList.size() ){
+ pos=i;
+ }
+
+ return pos;
+}
+
+string
+nlsrTokenizer::getTokenString(int from , int to){
+ string returnString;
+ if ( from >=0 && to < tokenList.size()){
+ int i=1;
+ for(std::list<string>::iterator it=tokenList.begin();
+ it!=tokenList.end();it++){
+ if( i >= from && i<= to ){
+ string oneToken((*it));
+ returnString+=seps;
+ returnString+=oneToken;
+
+ }
+ i++;
+ }
+ }
+
+ trim(returnString);
+ return returnString;
+}
+
+string
+nlsrTokenizer::getTokenString(int from){
+ string returnString;
+ if ( from >=0 && from < tokenList.size()){
+ int i=1;
+ for(std::list<string>::iterator it=tokenList.begin();
+ it!=tokenList.end();it++){
+ if( i >= from){
+ string oneToken((*it));
+ returnString+=seps;
+ returnString+=oneToken;
+
+ }
+ i++;
+ }
+ }
+
+ trim(returnString);
+ return returnString;
+}
+
+static bool
+tokenCompare(string& s1, string& s2){
+ return s1==s2;
+}
+
+bool
+nlsrTokenizer::doesTokenExist(string token){
+ std::list<string >::iterator it = std::find_if( tokenList.begin(),
+ tokenList.end(),
+ bind(&tokenCompare, _1 , token));
+
+ if( it != tokenList.end() ){
+ return true;
+ }
+
+ return false;
+}