|
Lines 1531-1547
Link Here
|
| 1531 |
return ent; |
1531 |
return ent; |
| 1532 |
} |
1532 |
} |
| 1533 |
|
1533 |
|
| 1534 |
void FullBtreeDict::store(const std::string &bla, int id) |
1534 |
void FullBtreeDict::store(const std::string &key, int id) |
| 1535 |
{ |
1535 |
{ |
| 1536 |
HCDBG(std::cerr << "storing " << bla << " id " << id << std::endl); |
1536 |
HCDBG(std::cerr << "so storing " << key << " id " << id << std::endl); |
| 1537 |
/* |
|
|
| 1538 |
if (logging) |
| 1539 |
log.write(key + " " + id + "\n"); |
| 1540 |
*/ |
| 1541 |
std::string key = bla; |
| 1542 |
int length = key.size(); |
| 1543 |
while(key.size() >= 250) |
| 1544 |
key = bla.substr(--length); |
| 1545 |
|
1537 |
|
| 1546 |
if (key.size() >= 250) |
1538 |
if (key.size() >= 250) |
| 1547 |
{ |
1539 |
{ |
|
Lines 2225-2233
Link Here
|
| 2225 |
} |
2217 |
} |
| 2226 |
} |
2218 |
} |
| 2227 |
|
2219 |
|
|
|
2220 |
namespace |
| 2221 |
{ |
| 2222 |
std::string cliptoken(const std::string &name) |
| 2223 |
{ |
| 2224 |
std::string key = name; |
| 2225 |
int length = key.size(); |
| 2226 |
while(key.size() >= 250) |
| 2227 |
key = name.substr(--length); |
| 2228 |
return key; |
| 2229 |
} |
| 2230 |
} |
| 2231 |
|
| 2228 |
int Index::intern(const std::string &name) |
2232 |
int Index::intern(const std::string &name) |
| 2229 |
{ |
2233 |
{ |
| 2230 |
IndexHashtable::const_iterator aIter = _cache.find(name); |
2234 |
std::string key = cliptoken(name); |
|
|
2235 |
IndexHashtable::const_iterator aIter = _cache.find(key); |
| 2231 |
if (aIter != _cache.end()) |
2236 |
if (aIter != _cache.end()) |
| 2232 |
return aIter->second; |
2237 |
return aIter->second; |
| 2233 |
else |
2238 |
else |
|
Lines 2235-2249
Link Here
|
| 2235 |
//Seeing as we always start off with an empty dictionary, |
2240 |
//Seeing as we always start off with an empty dictionary, |
| 2236 |
//our entries will always be in the _cache, so don't ever |
2241 |
//our entries will always be in the _cache, so don't ever |
| 2237 |
//search the underlying dictionary |
2242 |
//search the underlying dictionary |
| 2238 |
#if 0 |
|
|
| 2239 |
int id = _dict->fetch(name); |
| 2240 |
if (id == 0) |
| 2241 |
_dict->store(name, id = _freeID++); |
| 2242 |
#else |
| 2243 |
int id = _freeID++; |
2243 |
int id = _freeID++; |
| 2244 |
_dict->store(name, id); |
2244 |
_dict->store(key, id); |
| 2245 |
#endif |
2245 |
_cache.insert(IndexHashtable::value_type(key, id)).first->second = id; |
| 2246 |
_cache.insert(IndexHashtable::value_type(name, id)).first->second = id; |
|
|
| 2247 |
return id; |
2246 |
return id; |
| 2248 |
} |
2247 |
} |
| 2249 |
} |
2248 |
} |
|
Lines 3329-3336
Link Here
|
| 3329 |
{ |
3328 |
{ |
| 3330 |
#if !defined(SOLARIS) |
3329 |
#if !defined(SOLARIS) |
| 3331 |
delete bi; |
3330 |
delete bi; |
| 3332 |
#endif |
|
|
| 3333 |
ucnv_close(utf8); |
3331 |
ucnv_close(utf8); |
|
|
3332 |
#endif |
| 3334 |
} |
3333 |
} |
| 3335 |
|
3334 |
|
| 3336 |
void Tokenizer::setText(const xmlChar *text) |
3335 |
void Tokenizer::setText(const xmlChar *text) |