NepomukDaemons
clucenetokenizer.cpp File Reference
#include <CLucene/StdHeader.h>
#include "clucenetokenizer.h"

Go to the source code of this file.
Namespaces | |
namespace | Nepomuk |
Defines | |
#define | _CJK |
#define | _CONSUME_AS_LONG_AS(conditionFails) while (true) { ch = readChar(); if (ch==-1 || (!(conditionFails) || str.len >= LUCENE_MAX_WORD_LEN)) { break; } str.appendChar(ch);} |
#define | ALNUM (_istalnum(ch) != 0) |
#define | ALPHA (_istalpha((TCHAR)ch) != 0) |
#define | CONSUME_ALPHAS _CONSUME_AS_LONG_AS(ALPHA) |
#define | CONSUME_CJK _CONSUME_AS_LONG_AS(_CJK) |
#define | CONSUME_DIGITS _CONSUME_AS_LONG_AS(DIGIT) |
#define | CONSUME_WORD _CONSUME_AS_LONG_AS(ALNUM) |
#define | CONSUMED_NOTHING_OF_VALUE (rdPos == specialCharPos || (rdPos == specialCharPos+1 && ( SPACE || !(ALNUM || DOT || DASH || UNDERSCORE) ))) |
#define | CONTAINS_ANY(sb, ofThese) (_tcscspn(sb.getBuffer(), _T(ofThese)) != static_cast<size_t>(sb.len)) |
#define | DASH (ch == '-') |
#define | DECIMAL DOT |
#define | DIGIT (_istdigit(ch) != 0) |
#define | DOT (ch == '.') |
#define | EOS (ch==-1 || rd->Eos()) |
#define | NEGATIVE_SIGN_ DASH |
#define | RIGHTMOST(sb) (sb.getBuffer()[sb.len-1]) |
#define | RIGHTMOST_IS(sb, c) (RIGHTMOST(sb) == c) |
#define | SHAVE_RIGHTMOST(sb) (sb.getBuffer()[--sb.len] = '\0') |
#define | SPACE (_istspace((TCHAR)ch) != 0) |
#define | UNDERSCORE (ch == '_') |
Variables | |
const TCHAR ** | Nepomuk::tokenImage = tokenImageArray |
const TCHAR * | Nepomuk::tokenImageArray [] |
Define Documentation
#define _CJK |
Value:
( (ch>=0x3040 && ch<=0x318f) || \ (ch>=0x3300 && ch<=0x337f) || \ (ch>=0x3400 && ch<=0x3d2d) || \ (ch>=0x4e00 && ch<=0x9fff) || \ (ch>=0xf900 && ch<=0xfaff) || \ (ch>=0xac00 && ch<=0xd7af) )
Definition at line 66 of file clucenetokenizer.cpp.
#define _CONSUME_AS_LONG_AS | ( | conditionFails | ) | while (true) { ch = readChar(); if (ch==-1 || (!(conditionFails) || str.len >= LUCENE_MAX_WORD_LEN)) { break; } str.appendChar(ch);} |
Definition at line 84 of file clucenetokenizer.cpp.
#define ALNUM (_istalnum(ch) != 0) |
Definition at line 62 of file clucenetokenizer.cpp.
#define ALPHA (_istalpha((TCHAR)ch) != 0) |
Definition at line 61 of file clucenetokenizer.cpp.
#define CONSUME_ALPHAS _CONSUME_AS_LONG_AS(ALPHA) |
Definition at line 86 of file clucenetokenizer.cpp.
#define CONSUME_CJK _CONSUME_AS_LONG_AS(_CJK) |
Definition at line 98 of file clucenetokenizer.cpp.
#define CONSUME_DIGITS _CONSUME_AS_LONG_AS(DIGIT) |
Definition at line 88 of file clucenetokenizer.cpp.
#define CONSUME_WORD _CONSUME_AS_LONG_AS(ALNUM) |
Definition at line 93 of file clucenetokenizer.cpp.
#define CONSUMED_NOTHING_OF_VALUE (rdPos == specialCharPos || (rdPos == specialCharPos+1 && ( SPACE || !(ALNUM || DOT || DASH || UNDERSCORE) ))) |
Definition at line 107 of file clucenetokenizer.cpp.
#define CONTAINS_ANY | ( | sb, | |||
ofThese | ) | (_tcscspn(sb.getBuffer(), _T(ofThese)) != static_cast<size_t>(sb.len)) |
Definition at line 118 of file clucenetokenizer.cpp.
#define DASH (ch == '-') |
Definition at line 74 of file clucenetokenizer.cpp.
#define DECIMAL DOT |
Definition at line 80 of file clucenetokenizer.cpp.
#define DIGIT (_istdigit(ch) != 0) |
Definition at line 63 of file clucenetokenizer.cpp.
#define DOT (ch == '.') |
Definition at line 79 of file clucenetokenizer.cpp.
#define EOS (ch==-1 || rd->Eos()) |
Definition at line 59 of file clucenetokenizer.cpp.
#define NEGATIVE_SIGN_ DASH |
Definition at line 75 of file clucenetokenizer.cpp.
#define RIGHTMOST | ( | sb | ) | (sb.getBuffer()[sb.len-1]) |
Definition at line 109 of file clucenetokenizer.cpp.
#define RIGHTMOST_IS | ( | sb, | |||
c | ) | (RIGHTMOST(sb) == c) |
Definition at line 110 of file clucenetokenizer.cpp.
#define SHAVE_RIGHTMOST | ( | sb | ) | (sb.getBuffer()[--sb.len] = '\0') |
Definition at line 113 of file clucenetokenizer.cpp.
#define SPACE (_istspace((TCHAR)ch) != 0) |
Definition at line 60 of file clucenetokenizer.cpp.
#define UNDERSCORE (ch == '_') |
Definition at line 64 of file clucenetokenizer.cpp.