blob: 9916df66ad12edb3ef827e4669c0dda819f24337 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
#ifndef LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_
#define LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_
#include <vector>
#include <string>
#include <memory>
#include <assert.h>
namespace indexlib { namespace detail {
class tokenizer {
public:
virtual ~tokenizer() { }
std::vector<std::string> string_to_words( const char* str ) {
assert( str );
return do_string_to_words( str );
}
private:
virtual std::vector<std::string> do_string_to_words( const char* ) = 0;
};
std::unique_ptr<tokenizer> get_tokenizer( std::string );
}}
#endif /* LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_ */
|
