diff --git a/src/utility/nlsr_tokenizer.cpp b/src/utility/nlsr_tokenizer.cpp
new file mode 100644
index 0000000..809e395
--- /dev/null
+++ b/src/utility/nlsr_tokenizer.cpp
@@ -0,0 +1,110 @@
+#include <iostream>
+#include <boost/tokenizer.hpp>
+#include <boost/algorithm/string.hpp>
+#include <string>
+#include <algorithm>
+
+#include "nlsr_tokenizer.hpp"
+
+namespace nlsr
+{
+
+    using namespace std;
+    using namespace boost;
+
+    void
+    nlsrTokenizer::makeToken()
+    {
+        char_separator<char> sep(seps.c_str());
+        tokenizer< char_separator<char> >tokens(originalString, sep);
+        tokenizer< char_separator<char> >::iterator tok_iter = tokens.begin();
+        for ( ; tok_iter != tokens.end(); ++tok_iter)
+        {
+            string oneToken(*tok_iter);
+            trim(oneToken);
+            if(!oneToken.empty())
+            {
+                insertToken(oneToken);
+            }
+        }
+        firstToken=vTokenList[0];
+        makeRestOfTheLine();
+    }
+
+    void
+    nlsrTokenizer::insertToken(const string& token)
+    {
+        tokenList.push_back(token);
+        vTokenList.push_back(token);
+    }
+
+    int
+    nlsrTokenizer::getTokenPosition(string& token)
+    {
+        int pos=-1;
+        int i=0;
+        for(std::list<string>::iterator it=tokenList.begin();
+                it!=tokenList.end(); it++)
+        {
+            if( (*it) == token )
+            {
+                break;
+            }
+            i++;
+        }
+        if( i < tokenList.size() )
+        {
+            pos=i;
+        }
+        return pos;
+    }
+
+    string
+    nlsrTokenizer::getTokenString(int from , int to)
+    {
+        string returnString="";
+        if((from>=0 && to<tokenList.size()) &&
+                (to>=from && to <tokenList.size()))
+        {
+            for(int i=from; i<=to; i++)
+            {
+                returnString+=seps;
+                returnString+=vTokenList[i];
+            }
+        }
+        trim(returnString);
+        return returnString;
+    }
+
+    string
+    nlsrTokenizer::getTokenString(int from)
+    {
+        return getTokenString(from,tokenList.size()-1);
+    }
+
+    static bool
+    tokenCompare(string& s1, string& s2)
+    {
+        return s1==s2;
+    }
+
+    void
+    nlsrTokenizer::makeRestOfTheLine()
+    {
+        restOfTheLine=getTokenString(1);
+    }
+
+    bool
+    nlsrTokenizer::doesTokenExist(string token)
+    {
+        std::list<string >::iterator it = std::find_if( tokenList.begin(),
+                                          tokenList.end(),
+                                          bind(&tokenCompare, _1 , token));
+        if( it != tokenList.end() )
+        {
+            return true;
+        }
+        return false;
+    }
+
+}//namespace nlsr