| // (C) Copyright Gennadiy Rozental 2004-2008. |
| // Distributed under the Boost Software License, Version 1.0. |
| // (See accompanying file LICENSE_1_0.txt or copy at |
| // http://www.boost.org/LICENSE_1_0.txt) |
| |
| // See http://www.boost.org/libs/test for the library home page. |
| // |
| // File : $RCSfile$ |
| // |
| // Version : $Revision: 54633 $ |
| // |
| // Description : token iterator for string and range tokenization |
| // *************************************************************************** |
| |
| #ifndef BOOST_TOKEN_ITERATOR_HPP_071894GER |
| #define BOOST_TOKEN_ITERATOR_HPP_071894GER |
| |
| // Boost |
| #include <ndnboost/config.hpp> |
| #include <ndnboost/detail/workaround.hpp> |
| |
| #include <ndnboost/iterator/iterator_categories.hpp> |
| #include <ndnboost/iterator/iterator_traits.hpp> |
| |
| #include <ndnboost/test/utils/iterator/input_iterator_facade.hpp> |
| #include <ndnboost/test/utils/basic_cstring/basic_cstring.hpp> |
| #include <ndnboost/test/utils/named_params.hpp> |
| #include <ndnboost/test/utils/foreach.hpp> |
| |
| // STL |
| #include <iosfwd> |
| #include <cctype> |
| |
| #include <ndnboost/test/detail/suppress_warnings.hpp> |
| |
| //____________________________________________________________________________// |
| |
| #ifdef BOOST_NO_STDC_NAMESPACE |
| namespace std{ using ::ispunct; using ::isspace; } |
| #endif |
| |
| namespace ndnboost { |
| |
| namespace unit_test { |
| |
| // ************************************************************************** // |
| // ************** ti_delimeter_type ************** // |
| // ************************************************************************** // |
| |
| enum ti_delimeter_type { |
| dt_char, // character is delimeter if it among explicit list of some characters |
| dt_ispunct, // character is delimeter if it satisfies ispunct functor |
| dt_isspace, // character is delimeter if it satisfies isspace functor |
| dt_none // no character is delimeter |
| }; |
| |
| namespace ut_detail { |
| |
| // ************************************************************************** // |
| // ************** default_char_compare ************** // |
| // ************************************************************************** // |
| |
| template<typename CharT> |
| class default_char_compare { |
| public: |
| bool operator()( CharT c1, CharT c2 ) |
| { |
| #ifdef BOOST_CLASSIC_IOSTREAMS |
| return std::string_char_traits<CharT>::eq( c1, c2 ); |
| #else |
| return std::char_traits<CharT>::eq( c1, c2 ); |
| #endif |
| } |
| }; |
| |
| // ************************************************************************** // |
| // ************** delim_policy ************** // |
| // ************************************************************************** // |
| |
| template<typename CharT,typename CharCompare> |
| class delim_policy { |
| typedef basic_cstring<CharT const> cstring; |
| public: |
| // Constructor |
| explicit delim_policy( ti_delimeter_type t = dt_char, cstring d = cstring() ) |
| : m_type( t ) |
| { |
| set_delimeters( d ); |
| } |
| |
| void set_delimeters( ti_delimeter_type t ) { m_type = t; } |
| template<typename Src> |
| void set_delimeters( Src d ) |
| { |
| nfp::optionally_assign( m_delimeters, d ); |
| |
| if( !m_delimeters.is_empty() ) |
| m_type = dt_char; |
| } |
| |
| bool operator()( CharT c ) |
| { |
| switch( m_type ) { |
| case dt_char: { |
| BOOST_TEST_FOREACH( CharT, delim, m_delimeters ) |
| if( CharCompare()( delim, c ) ) |
| return true; |
| |
| return false; |
| } |
| case dt_ispunct: |
| return (std::ispunct)( c ) != 0; |
| case dt_isspace: |
| return (std::isspace)( c ) != 0; |
| case dt_none: |
| return false; |
| } |
| |
| return false; |
| } |
| |
| private: |
| // Data members |
| cstring m_delimeters; |
| ti_delimeter_type m_type; |
| }; |
| |
| // ************************************************************************** // |
| // ************** token_assigner ************** // |
| // ************************************************************************** // |
| |
| template<typename TraversalTag> |
| struct token_assigner { |
| #if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 ) |
| template<typename Iterator, typename C, typename T> |
| static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t ) |
| { for( ; b != e; ++b ) t += *b; } |
| |
| template<typename Iterator, typename C> |
| static void assign( Iterator b, Iterator e, basic_cstring<C>& t ) { t.assign( b, e ); } |
| #else |
| template<typename Iterator, typename Token> |
| static void assign( Iterator b, Iterator e, Token& t ) { t.assign( b, e ); } |
| #endif |
| template<typename Iterator, typename Token> |
| static void append_move( Iterator& b, Token& ) { ++b; } |
| }; |
| |
| //____________________________________________________________________________// |
| |
| template<> |
| struct token_assigner<single_pass_traversal_tag> { |
| template<typename Iterator, typename Token> |
| static void assign( Iterator b, Iterator e, Token& t ) {} |
| |
| template<typename Iterator, typename Token> |
| static void append_move( Iterator& b, Token& t ) { t += *b; ++b; } |
| }; |
| |
| } // namespace ut_detail |
| |
| // ************************************************************************** // |
| // ************** modifiers ************** // |
| // ************************************************************************** // |
| |
| namespace { |
| nfp::keyword<struct dropped_delimeters_t > dropped_delimeters; |
| nfp::keyword<struct kept_delimeters_t > kept_delimeters; |
| nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens; |
| nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens; |
| } |
| |
| // ************************************************************************** // |
| // ************** token_iterator_base ************** // |
| // ************************************************************************** // |
| |
| template<typename Derived, |
| typename CharT, |
| typename CharCompare = ut_detail::default_char_compare<CharT>, |
| typename ValueType = basic_cstring<CharT const>, |
| typename Reference = basic_cstring<CharT const>, |
| typename Traversal = forward_traversal_tag> |
| class token_iterator_base |
| : public input_iterator_facade<Derived,ValueType,Reference,Traversal> { |
| typedef basic_cstring<CharT const> cstring; |
| typedef ut_detail::delim_policy<CharT,CharCompare> delim_policy; |
| typedef input_iterator_facade<Derived,ValueType,Reference,Traversal> base; |
| |
| protected: |
| // Constructor |
| explicit token_iterator_base() |
| : m_is_dropped( dt_isspace ) |
| , m_is_kept( dt_ispunct ) |
| , m_keep_empty_tokens( false ) |
| , m_tokens_left( static_cast<std::size_t>(-1) ) |
| , m_token_produced( false ) |
| { |
| } |
| |
| template<typename Modifier> |
| void |
| apply_modifier( Modifier const& m ) |
| { |
| if( m.has( dropped_delimeters ) ) |
| m_is_dropped.set_delimeters( m[dropped_delimeters] ); |
| |
| if( m.has( kept_delimeters ) ) |
| m_is_kept.set_delimeters( m[kept_delimeters] ); |
| |
| if( m.has( keep_empty_tokens ) ) |
| m_keep_empty_tokens = true; |
| |
| nfp::optionally_assign( m_tokens_left, m, max_tokens ); |
| } |
| |
| template<typename Iter> |
| bool get( Iter& begin, Iter end ) |
| { |
| typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner; |
| Iter check_point; |
| |
| this->m_value.clear(); |
| |
| if( !m_keep_empty_tokens ) { |
| while( begin != end && m_is_dropped( *begin ) ) |
| ++begin; |
| |
| if( begin == end ) |
| return false; |
| |
| check_point = begin; |
| |
| if( m_tokens_left == 1 ) |
| while( begin != end ) |
| Assigner::append_move( begin, this->m_value ); |
| else if( m_is_kept( *begin ) ) |
| Assigner::append_move( begin, this->m_value ); |
| else |
| while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) ) |
| Assigner::append_move( begin, this->m_value ); |
| |
| --m_tokens_left; |
| } |
| else { // m_keep_empty_tokens is true |
| check_point = begin; |
| |
| if( begin == end ) { |
| if( m_token_produced ) |
| return false; |
| |
| m_token_produced = true; |
| } |
| if( m_is_kept( *begin ) ) { |
| if( m_token_produced ) |
| Assigner::append_move( begin, this->m_value ); |
| |
| m_token_produced = !m_token_produced; |
| } |
| else if( !m_token_produced && m_is_dropped( *begin ) ) |
| m_token_produced = true; |
| else { |
| if( m_is_dropped( *begin ) ) |
| check_point = ++begin; |
| |
| while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) ) |
| Assigner::append_move( begin, this->m_value ); |
| |
| m_token_produced = true; |
| } |
| } |
| |
| Assigner::assign( check_point, begin, this->m_value ); |
| |
| return true; |
| } |
| |
| private: |
| // Data members |
| delim_policy m_is_dropped; |
| delim_policy m_is_kept; |
| bool m_keep_empty_tokens; |
| std::size_t m_tokens_left; |
| bool m_token_produced; |
| }; |
| |
| // ************************************************************************** // |
| // ************** basic_string_token_iterator ************** // |
| // ************************************************************************** // |
| |
| template<typename CharT, |
| typename CharCompare = ut_detail::default_char_compare<CharT> > |
| class basic_string_token_iterator |
| : public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> { |
| typedef basic_cstring<CharT const> cstring; |
| typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base; |
| public: |
| explicit basic_string_token_iterator() {} |
| explicit basic_string_token_iterator( cstring src ) |
| : m_src( src ) |
| { |
| this->init(); |
| } |
| |
| template<typename Src, typename Modifier> |
| basic_string_token_iterator( Src src, Modifier const& m ) |
| : m_src( src ) |
| { |
| this->apply_modifier( m ); |
| |
| this->init(); |
| } |
| |
| private: |
| friend class input_iterator_core_access; |
| |
| // input iterator implementation |
| bool get() |
| { |
| typename cstring::iterator begin = m_src.begin(); |
| bool res = base::get( begin, m_src.end() ); |
| |
| m_src.assign( begin, m_src.end() ); |
| |
| return res; |
| } |
| |
| // Data members |
| cstring m_src; |
| }; |
| |
| typedef basic_string_token_iterator<char> string_token_iterator; |
| typedef basic_string_token_iterator<wchar_t> wstring_token_iterator; |
| |
| // ************************************************************************** // |
| // ************** range_token_iterator ************** // |
| // ************************************************************************** // |
| |
| template<typename Iter, |
| typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>, |
| typename ValueType = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>, |
| typename Reference = ValueType const&> |
| class range_token_iterator |
| : public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>, |
| typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> { |
| typedef basic_cstring<typename ValueType::value_type> cstring; |
| typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>, |
| typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base; |
| public: |
| explicit range_token_iterator() {} |
| explicit range_token_iterator( Iter begin, Iter end = Iter() ) |
| : m_begin( begin ), m_end( end ) |
| { |
| this->init(); |
| } |
| range_token_iterator( range_token_iterator const& rhs ) |
| : base( rhs ) |
| { |
| if( this->m_valid ) { |
| m_begin = rhs.m_begin; |
| m_end = rhs.m_end; |
| } |
| } |
| |
| template<typename Modifier> |
| range_token_iterator( Iter begin, Iter end, Modifier const& m ) |
| : m_begin( begin ), m_end( end ) |
| { |
| this->apply_modifier( m ); |
| |
| this->init(); |
| } |
| |
| private: |
| friend class input_iterator_core_access; |
| |
| // input iterator implementation |
| bool get() |
| { |
| return base::get( m_begin, m_end ); |
| } |
| |
| // Data members |
| Iter m_begin; |
| Iter m_end; |
| }; |
| |
| // ************************************************************************** // |
| // ************** make_range_token_iterator ************** // |
| // ************************************************************************** // |
| |
| template<typename Iter> |
| inline range_token_iterator<Iter> |
| make_range_token_iterator( Iter begin, Iter end = Iter() ) |
| { |
| return range_token_iterator<Iter>( begin, end ); |
| } |
| |
| //____________________________________________________________________________// |
| |
| template<typename Iter,typename Modifier> |
| inline range_token_iterator<Iter> |
| make_range_token_iterator( Iter begin, Iter end, Modifier const& m ) |
| { |
| return range_token_iterator<Iter>( begin, end, m ); |
| } |
| |
| //____________________________________________________________________________// |
| |
| } // namespace unit_test |
| |
| } // namespace ndnboost |
| |
| //____________________________________________________________________________// |
| |
| #include <ndnboost/test/detail/enable_warnings.hpp> |
| |
| #endif // BOOST_TOKEN_ITERATOR_HPP_071894GER |
| |