src/utility/nlsr_tokenizer.cpp

#include <iostream>
#include <boost/tokenizer.hpp>
#include <boost/algorithm/string.hpp>
#include <string>
#include <algorithm>

#include "nlsr_tokenizer.hpp"

namespace nlsr
{

  using namespace std;
  using namespace boost;

  void
  nlsrTokenizer::makeToken()
  {
    char_separator<char> sep(seps.c_str());
    tokenizer< char_separator<char> >tokens(originalString, sep);
    tokenizer< char_separator<char> >::iterator tok_iter = tokens.begin();
    for ( ; tok_iter != tokens.end(); ++tok_iter)
    {
      string oneToken(*tok_iter);
      trim(oneToken);
      if(!oneToken.empty())
      {
        insertToken(oneToken);
      }
    }
    firstToken=vTokenList[0];
    makeRestOfTheLine();
  }

  void
  nlsrTokenizer::insertToken(const string& token)
  {
    tokenList.push_back(token);
    vTokenList.push_back(token);
  }

  int
  nlsrTokenizer::getTokenPosition(string& token)
  {
    int pos=-1;
    int i=0;
    for(std::list<string>::iterator it=tokenList.begin();
        it!=tokenList.end(); it++)
    {
      if( (*it) == token )
      {
        break;
      }
      i++;
    }
    if( i < tokenList.size() )
    {
      pos=i;
    }
    return pos;
  }

  string
  nlsrTokenizer::getTokenString(int from , int to)
  {
    string returnString="";
    if((from>=0 && to<tokenList.size()) &&
        (to>=from && to <tokenList.size()))
    {
      for(int i=from; i<=to; i++)
      {
        returnString+=seps;
        returnString+=vTokenList[i];
      }
    }
    trim(returnString);
    return returnString;
  }

  string
  nlsrTokenizer::getTokenString(int from)
  {
    return getTokenString(from,tokenList.size()-1);
  }

  static bool
  tokenCompare(string& s1, string& s2)
  {
    return s1==s2;
  }

  void
  nlsrTokenizer::makeRestOfTheLine()
  {
    restOfTheLine=getTokenString(1);
  }

  bool
  nlsrTokenizer::doesTokenExist(string token)
  {
    std::list<string >::iterator it = std::find_if( tokenList.begin(),
                                      tokenList.end(),
                                      bind(&tokenCompare, _1 , token));
    if( it != tokenList.end() )
    {
      return true;
    }
    return false;
  }

}//namespace nlsr