PostgreSQL Source Code: src/backend/tsearch/ts_parse.c Source File

PostgreSQL Source Code git master
ts_parse.c
Go to the documentation of this file.
1/*-------------------------------------------------------------------------
2 *
3 * ts_parse.c
4 * main parse functions for tsearch
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/tsearch/ts_parse.c
11 *
12 *-------------------------------------------------------------------------
13 */
14
15#include "postgres.h"
16
17#include "tsearch/ts_cache.h"
18#include "tsearch/ts_utils.h"
19#include "varatt.h"
20
21 #define IGNORE_LONGLEXEME 1
22
23/*
24 * Lexize subsystem
25 */
26
27 typedef struct ParsedLex
28{
29 int type;
30 char *lemm;
31 int lenlemm;
32 struct ParsedLex *next;
33 } ParsedLex;
34
35 typedef struct ListParsedLex
36{
37 ParsedLex *head;
38 ParsedLex *tail;
39 } ListParsedLex;
40
41 typedef struct
42{
43 TSConfigCacheEntry *cfg;
44 Oid curDictId;
45 int posDict;
46 DictSubState dictState;
47 ParsedLex *curSub;
48 ListParsedLex towork; /* current list to work */
49 ListParsedLex waste; /* list of lexemes that already lexized */
50
51 /*
52 * fields to store last variant to lexize (basically, thesaurus or similar
53 * to, which wants several lexemes
54 */
55
56 ParsedLex *lastRes;
57 TSLexeme *tmpRes;
58} LexizeData;
59
60static void
61 LexizeInit(LexizeData *ld, TSConfigCacheEntry *cfg)
62{
63 ld->cfg = cfg;
64 ld->curDictId = InvalidOid;
65 ld->posDict = 0;
66 ld->towork.head = ld->towork.tail = ld->curSub = NULL;
67 ld->waste.head = ld->waste.tail = NULL;
68 ld->lastRes = NULL;
69 ld->tmpRes = NULL;
70}
71
72static void
73 LPLAddTail(ListParsedLex *list, ParsedLex *newpl)
74{
75 if (list->tail)
76 {
77 list->tail->next = newpl;
78 list->tail = newpl;
79 }
80 else
81 list->head = list->tail = newpl;
82 newpl->next = NULL;
83}
84
85static ParsedLex *
86 LPLRemoveHead(ListParsedLex *list)
87{
88 ParsedLex *res = list->head;
89
90 if (list->head)
91 list->head = list->head->next;
92
93 if (list->head == NULL)
94 list->tail = NULL;
95
96 return res;
97}
98
99static void
100 LexizeAddLemm(LexizeData *ld, int type, char *lemm, int lenlemm)
101{
102 ParsedLex *newpl = (ParsedLex *) palloc(sizeof(ParsedLex));
103
104 newpl->type = type;
105 newpl->lemm = lemm;
106 newpl->lenlemm = lenlemm;
107 LPLAddTail(&ld->towork, newpl);
108 ld->curSub = ld->towork.tail;
109}
110
111static void
112 RemoveHead(LexizeData *ld)
113{
114 LPLAddTail(&ld->waste, LPLRemoveHead(&ld->towork));
115
116 ld->posDict = 0;
117}
118
119static void
120 setCorrLex(LexizeData *ld, ParsedLex **correspondLexem)
121{
122 if (correspondLexem)
123 {
124 *correspondLexem = ld->waste.head;
125 }
126 else
127 {
128 ParsedLex *tmp,
129 *ptr = ld->waste.head;
130
131 while (ptr)
132 {
133 tmp = ptr->next;
134 pfree(ptr);
135 ptr = tmp;
136 }
137 }
138 ld->waste.head = ld->waste.tail = NULL;
139}
140
141static void
142 moveToWaste(LexizeData *ld, ParsedLex *stop)
143{
144 bool go = true;
145
146 while (ld->towork.head && go)
147 {
148 if (ld->towork.head == stop)
149 {
150 ld->curSub = stop->next;
151 go = false;
152 }
153 RemoveHead(ld);
154 }
155}
156
157static void
158 setNewTmpRes(LexizeData *ld, ParsedLex *lex, TSLexeme *res)
159{
160 if (ld->tmpRes)
161 {
162 TSLexeme *ptr;
163
164 for (ptr = ld->tmpRes; ptr->lexeme; ptr++)
165 pfree(ptr->lexeme);
166 pfree(ld->tmpRes);
167 }
168 ld->tmpRes = res;
169 ld->lastRes = lex;
170}
171
172static TSLexeme *
173 LexizeExec(LexizeData *ld, ParsedLex **correspondLexem)
174{
175 int i;
176 ListDictionary *map;
177 TSDictionaryCacheEntry *dict;
178 TSLexeme *res;
179
180 if (ld->curDictId == InvalidOid)
181 {
182 /*
183 * usual mode: dictionary wants only one word, but we should keep in
184 * mind that we should go through all stack
185 */
186
187 while (ld->towork.head)
188 {
189 ParsedLex *curVal = ld->towork.head;
190 char *curValLemm = curVal->lemm;
191 int curValLenLemm = curVal->lenlemm;
192
193 map = ld->cfg->map + curVal->type;
194
195 if (curVal->type == 0 || curVal->type >= ld->cfg->lenmap || map->len == 0)
196 {
197 /* skip this type of lexeme */
198 RemoveHead(ld);
199 continue;
200 }
201
202 for (i = ld->posDict; i < map->len; i++)
203 {
204 dict = lookup_ts_dictionary_cache(map->dictIds[i]);
205
206 ld->dictState.isend = ld->dictState.getnext = false;
207 ld->dictState.private_state = NULL;
208 res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize),
209 PointerGetDatum(dict->dictData),
210 PointerGetDatum(curValLemm),
211 Int32GetDatum(curValLenLemm),
212 PointerGetDatum(&ld->dictState)));
213
214 if (ld->dictState.getnext)
215 {
216 /*
217 * dictionary wants next word, so setup and store current
218 * position and go to multiword mode
219 */
220
221 ld->curDictId = map->dictIds[i];
222 ld->posDict = i + 1;
223 ld->curSub = curVal->next;
224 if (res)
225 setNewTmpRes(ld, curVal, res);
226 return LexizeExec(ld, correspondLexem);
227 }
228
229 if (!res) /* dictionary doesn't know this lexeme */
230 continue;
231
232 if (res->flags & TSL_FILTER)
233 {
234 curValLemm = res->lexeme;
235 curValLenLemm = strlen(res->lexeme);
236 continue;
237 }
238
239 RemoveHead(ld);
240 setCorrLex(ld, correspondLexem);
241 return res;
242 }
243
244 RemoveHead(ld);
245 }
246 }
247 else
248 { /* curDictId is valid */
249 dict = lookup_ts_dictionary_cache(ld->curDictId);
250
251 /*
252 * Dictionary ld->curDictId asks us about following words
253 */
254
255 while (ld->curSub)
256 {
257 ParsedLex *curVal = ld->curSub;
258
259 map = ld->cfg->map + curVal->type;
260
261 if (curVal->type != 0)
262 {
263 bool dictExists = false;
264
265 if (curVal->type >= ld->cfg->lenmap || map->len == 0)
266 {
267 /* skip this type of lexeme */
268 ld->curSub = curVal->next;
269 continue;
270 }
271
272 /*
273 * We should be sure that current type of lexeme is recognized
274 * by our dictionary: we just check is it exist in list of
275 * dictionaries ?
276 */
277 for (i = 0; i < map->len && !dictExists; i++)
278 if (ld->curDictId == map->dictIds[i])
279 dictExists = true;
280
281 if (!dictExists)
282 {
283 /*
284 * Dictionary can't work with current type of lexeme,
285 * return to basic mode and redo all stored lexemes
286 */
287 ld->curDictId = InvalidOid;
288 return LexizeExec(ld, correspondLexem);
289 }
290 }
291
292 ld->dictState.isend = (curVal->type == 0);
293 ld->dictState.getnext = false;
294
295 res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize),
296 PointerGetDatum(dict->dictData),
297 PointerGetDatum(curVal->lemm),
298 Int32GetDatum(curVal->lenlemm),
299 PointerGetDatum(&ld->dictState)));
300
301 if (ld->dictState.getnext)
302 {
303 /* Dictionary wants one more */
304 ld->curSub = curVal->next;
305 if (res)
306 setNewTmpRes(ld, curVal, res);
307 continue;
308 }
309
310 if (res || ld->tmpRes)
311 {
312 /*
313 * Dictionary normalizes lexemes, so we remove from stack all
314 * used lexemes, return to basic mode and redo end of stack
315 * (if it exists)
316 */
317 if (res)
318 {
319 moveToWaste(ld, ld->curSub);
320 }
321 else
322 {
323 res = ld->tmpRes;
324 moveToWaste(ld, ld->lastRes);
325 }
326
327 /* reset to initial state */
328 ld->curDictId = InvalidOid;
329 ld->posDict = 0;
330 ld->lastRes = NULL;
331 ld->tmpRes = NULL;
332 setCorrLex(ld, correspondLexem);
333 return res;
334 }
335
336 /*
337 * Dict don't want next lexem and didn't recognize anything, redo
338 * from ld->towork.head
339 */
340 ld->curDictId = InvalidOid;
341 return LexizeExec(ld, correspondLexem);
342 }
343 }
344
345 setCorrLex(ld, correspondLexem);
346 return NULL;
347}
348
349/*
350 * Parse string and lexize words.
351 *
352 * prs will be filled in.
353 */
354void
355 parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen)
356{
357 int type,
358 lenlemm = 0; /* silence compiler warning */
359 char *lemm = NULL;
360 LexizeData ldata;
361 TSLexeme *norms;
362 TSConfigCacheEntry *cfg;
363 TSParserCacheEntry *prsobj;
364 void *prsdata;
365
366 cfg = lookup_ts_config_cache(cfgId);
367 prsobj = lookup_ts_parser_cache(cfg->prsId);
368
369 prsdata = DatumGetPointer(FunctionCall2(&prsobj->prsstart,
370 PointerGetDatum(buf),
371 Int32GetDatum(buflen)));
372
373 LexizeInit(&ldata, cfg);
374
375 do
376 {
377 type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken),
378 PointerGetDatum(prsdata),
379 PointerGetDatum(&lemm),
380 PointerGetDatum(&lenlemm)));
381
382 if (type > 0 && lenlemm >= MAXSTRLEN)
383 {
384#ifdef IGNORE_LONGLEXEME
385 ereport(NOTICE,
386 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
387 errmsg("word is too long to be indexed"),
388 errdetail("Words longer than %d characters are ignored.",
389 MAXSTRLEN)));
390 continue;
391#else
392 ereport(ERROR,
393 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
394 errmsg("word is too long to be indexed"),
395 errdetail("Words longer than %d characters are ignored.",
396 MAXSTRLEN)));
397#endif
398 }
399
400 LexizeAddLemm(&ldata, type, lemm, lenlemm);
401
402 while ((norms = LexizeExec(&ldata, NULL)) != NULL)
403 {
404 TSLexeme *ptr = norms;
405
406 prs->pos++; /* set pos */
407
408 while (ptr->lexeme)
409 {
410 if (prs->curwords == prs->lenwords)
411 {
412 prs->lenwords *= 2;
413 prs->words = (ParsedWord *) repalloc(prs->words, prs->lenwords * sizeof(ParsedWord));
414 }
415
416 if (ptr->flags & TSL_ADDPOS)
417 prs->pos++;
418 prs->words[prs->curwords].len = strlen(ptr->lexeme);
419 prs->words[prs->curwords].word = ptr->lexeme;
420 prs->words[prs->curwords].nvariant = ptr->nvariant;
421 prs->words[prs->curwords].flags = ptr->flags & TSL_PREFIX;
422 prs->words[prs->curwords].alen = 0;
423 prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos);
424 ptr++;
425 prs->curwords++;
426 }
427 pfree(norms);
428 }
429 } while (type > 0);
430
431 FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata));
432}
433
434/*
435 * Headline framework
436 */
437
438/* Add a word to prs->words[] */
439static void
440 hladdword(HeadlineParsedText *prs, char *buf, int buflen, int type)
441{
442 if (prs->curwords >= prs->lenwords)
443 {
444 prs->lenwords *= 2;
445 prs->words = (HeadlineWordEntry *) repalloc(prs->words, prs->lenwords * sizeof(HeadlineWordEntry));
446 }
447 memset(&(prs->words[prs->curwords]), 0, sizeof(HeadlineWordEntry));
448 prs->words[prs->curwords].type = (uint8) type;
449 prs->words[prs->curwords].len = buflen;
450 prs->words[prs->curwords].word = palloc(buflen);
451 memcpy(prs->words[prs->curwords].word, buf, buflen);
452 prs->curwords++;
453}
454
455/*
456 * Add pos and matching-query-item data to the just-added word.
457 * Here, buf/buflen represent a processed lexeme, not raw token text.
458 *
459 * If the query contains more than one matching item, we replicate
460 * the last-added word so that each item can be pointed to. The
461 * duplicate entries are marked with repeated = 1.
462 */
463static void
464 hlfinditem(HeadlineParsedText *prs, TSQuery query, int32 pos, char *buf, int buflen)
465{
466 int i;
467 QueryItem *item = GETQUERY(query);
468 HeadlineWordEntry *word;
469
470 while (prs->curwords + query->size >= prs->lenwords)
471 {
472 prs->lenwords *= 2;
473 prs->words = (HeadlineWordEntry *) repalloc(prs->words, prs->lenwords * sizeof(HeadlineWordEntry));
474 }
475
476 word = &(prs->words[prs->curwords - 1]);
477 word->pos = LIMITPOS(pos);
478 for (i = 0; i < query->size; i++)
479 {
480 if (item->type == QI_VAL &&
481 tsCompareString(GETOPERAND(query) + item->qoperand.distance, item->qoperand.length,
482 buf, buflen, item->qoperand.prefix) == 0)
483 {
484 if (word->item)
485 {
486 memcpy(&(prs->words[prs->curwords]), word, sizeof(HeadlineWordEntry));
487 prs->words[prs->curwords].item = &item->qoperand;
488 prs->words[prs->curwords].repeated = 1;
489 prs->curwords++;
490 }
491 else
492 word->item = &item->qoperand;
493 }
494 item++;
495 }
496}
497
498static void
499 addHLParsedLex(HeadlineParsedText *prs, TSQuery query, ParsedLex *lexs, TSLexeme *norms)
500{
501 ParsedLex *tmplexs;
502 TSLexeme *ptr;
503 int32 savedpos;
504
505 while (lexs)
506 {
507 if (lexs->type > 0)
508 hladdword(prs, lexs->lemm, lexs->lenlemm, lexs->type);
509
510 ptr = norms;
511 savedpos = prs->vectorpos;
512 while (ptr && ptr->lexeme)
513 {
514 if (ptr->flags & TSL_ADDPOS)
515 savedpos++;
516 hlfinditem(prs, query, savedpos, ptr->lexeme, strlen(ptr->lexeme));
517 ptr++;
518 }
519
520 tmplexs = lexs->next;
521 pfree(lexs);
522 lexs = tmplexs;
523 }
524
525 if (norms)
526 {
527 ptr = norms;
528 while (ptr->lexeme)
529 {
530 if (ptr->flags & TSL_ADDPOS)
531 prs->vectorpos++;
532 pfree(ptr->lexeme);
533 ptr++;
534 }
535 pfree(norms);
536 }
537}
538
539void
540 hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int buflen)
541{
542 int type,
543 lenlemm = 0; /* silence compiler warning */
544 char *lemm = NULL;
545 LexizeData ldata;
546 TSLexeme *norms;
547 ParsedLex *lexs;
548 TSConfigCacheEntry *cfg;
549 TSParserCacheEntry *prsobj;
550 void *prsdata;
551
552 cfg = lookup_ts_config_cache(cfgId);
553 prsobj = lookup_ts_parser_cache(cfg->prsId);
554
555 prsdata = DatumGetPointer(FunctionCall2(&(prsobj->prsstart),
556 PointerGetDatum(buf),
557 Int32GetDatum(buflen)));
558
559 LexizeInit(&ldata, cfg);
560
561 do
562 {
563 type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken),
564 PointerGetDatum(prsdata),
565 PointerGetDatum(&lemm),
566 PointerGetDatum(&lenlemm)));
567
568 if (type > 0 && lenlemm >= MAXSTRLEN)
569 {
570#ifdef IGNORE_LONGLEXEME
571 ereport(NOTICE,
572 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
573 errmsg("word is too long to be indexed"),
574 errdetail("Words longer than %d characters are ignored.",
575 MAXSTRLEN)));
576 continue;
577#else
578 ereport(ERROR,
579 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
580 errmsg("word is too long to be indexed"),
581 errdetail("Words longer than %d characters are ignored.",
582 MAXSTRLEN)));
583#endif
584 }
585
586 LexizeAddLemm(&ldata, type, lemm, lenlemm);
587
588 do
589 {
590 if ((norms = LexizeExec(&ldata, &lexs)) != NULL)
591 {
592 prs->vectorpos++;
593 addHLParsedLex(prs, query, lexs, norms);
594 }
595 else
596 addHLParsedLex(prs, query, lexs, NULL);
597 } while (norms);
598 } while (type > 0);
599
600 FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata));
601}
602
603/*
604 * Generate the headline, as a text object, from HeadlineParsedText.
605 */
606text *
607 generateHeadline(HeadlineParsedText *prs)
608{
609 text *out;
610 char *ptr;
611 int len = 128;
612 int numfragments = 0;
613 int16 infrag = 0;
614
615 HeadlineWordEntry *wrd = prs->words;
616
617 out = (text *) palloc(len);
618 ptr = ((char *) out) + VARHDRSZ;
619
620 while (wrd - prs->words < prs->curwords)
621 {
622 while (wrd->len + prs->stopsellen + prs->startsellen + prs->fragdelimlen + (ptr - ((char *) out)) >= len)
623 {
624 int dist = ptr - ((char *) out);
625
626 len *= 2;
627 out = (text *) repalloc(out, len);
628 ptr = ((char *) out) + dist;
629 }
630
631 if (wrd->in && !wrd->repeated)
632 {
633 if (!infrag)
634 {
635
636 /* start of a new fragment */
637 infrag = 1;
638 numfragments++;
639 /* add a fragment delimiter if this is after the first one */
640 if (numfragments > 1)
641 {
642 memcpy(ptr, prs->fragdelim, prs->fragdelimlen);
643 ptr += prs->fragdelimlen;
644 }
645 }
646 if (wrd->replace)
647 {
648 *ptr = ' ';
649 ptr++;
650 }
651 else if (!wrd->skip)
652 {
653 if (wrd->selected)
654 {
655 memcpy(ptr, prs->startsel, prs->startsellen);
656 ptr += prs->startsellen;
657 }
658 memcpy(ptr, wrd->word, wrd->len);
659 ptr += wrd->len;
660 if (wrd->selected)
661 {
662 memcpy(ptr, prs->stopsel, prs->stopsellen);
663 ptr += prs->stopsellen;
664 }
665 }
666 }
667 else if (!wrd->repeated)
668 {
669 if (infrag)
670 infrag = 0;
671 pfree(wrd->word);
672 }
673
674 wrd++;
675 }
676
677 SET_VARSIZE(out, ptr - ((char *) out));
678 return out;
679}
#define GETQUERY(x)
Definition: _int.h:157
uint8_t uint8
Definition: c.h:536
#define VARHDRSZ
Definition: c.h:697
int16_t int16
Definition: c.h:533
int32_t int32
Definition: c.h:534
int errdetail(const char *fmt,...)
Definition: elog.c:1207
int errcode(int sqlerrcode)
Definition: elog.c:854
int errmsg(const char *fmt,...)
Definition: elog.c:1071
#define ERROR
Definition: elog.h:39
#define NOTICE
Definition: elog.h:35
#define ereport(elevel,...)
Definition: elog.h:150
#define FunctionCall4(flinfo, arg1, arg2, arg3, arg4)
Definition: fmgr.h:706
#define FunctionCall1(flinfo, arg1)
Definition: fmgr.h:700
#define FunctionCall2(flinfo, arg1, arg2)
Definition: fmgr.h:702
#define FunctionCall3(flinfo, arg1, arg2, arg3)
Definition: fmgr.h:704
i
int i
Definition: isn.c:77
#define GETOPERAND(x)
Definition: ltree.h:165
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1610
void pfree(void *pointer)
Definition: mcxt.c:1594
void * palloc(Size size)
Definition: mcxt.c:1365
const void size_t len
static char * buf
Definition: pg_test_fsync.c:72
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:332
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:322
static Datum Int32GetDatum(int32 X)
Definition: postgres.h:222
static int32 DatumGetInt32(Datum X)
Definition: postgres.h:212
#define InvalidOid
Definition: postgres_ext.h:37
unsigned int Oid
Definition: postgres_ext.h:32
static void word(struct vars *v, int dir, struct state *lp, struct state *rp)
Definition: regcomp.c:1476
void * private_state
Definition: ts_public.h:155
bool isend
Definition: ts_public.h:152
bool getnext
Definition: ts_public.h:154
HeadlineWordEntry * words
Definition: ts_public.h:76
char * stopsel
Definition: ts_public.h:84
int16 startsellen
Definition: ts_public.h:86
int32 curwords
Definition: ts_public.h:78
char * fragdelim
Definition: ts_public.h:85
int16 stopsellen
Definition: ts_public.h:87
int32 lenwords
Definition: ts_public.h:77
char * startsel
Definition: ts_public.h:83
int32 vectorpos
Definition: ts_public.h:79
int16 fragdelimlen
Definition: ts_public.h:88
uint32 skip
Definition: ts_public.h:64
uint32 type
Definition: ts_public.h:66
uint32 repeated
Definition: ts_public.h:63
uint32 replace
Definition: ts_public.h:62
uint32 len
Definition: ts_public.h:67
char * word
Definition: ts_public.h:69
uint32 selected
Definition: ts_public.h:60
QueryOperand * item
Definition: ts_public.h:70
TSLexeme * tmpRes
Definition: ts_parse.c:57
Oid curDictId
Definition: ts_parse.c:44
ListParsedLex towork
Definition: ts_parse.c:48
ParsedLex * curSub
Definition: ts_parse.c:47
ParsedLex * lastRes
Definition: ts_parse.c:56
int posDict
Definition: ts_parse.c:45
ListParsedLex waste
Definition: ts_parse.c:49
DictSubState dictState
Definition: ts_parse.c:46
TSConfigCacheEntry * cfg
Definition: ts_parse.c:43
Oid * dictIds
Definition: ts_cache.h:68
ParsedLex * tail
Definition: ts_parse.c:38
ParsedLex * head
Definition: ts_parse.c:37
char * lemm
Definition: ts_parse.c:30
int lenlemm
Definition: ts_parse.c:31
struct ParsedLex * next
Definition: ts_parse.c:32
int type
Definition: ts_parse.c:29
int32 pos
Definition: ts_utils.h:107
int32 lenwords
Definition: ts_utils.h:105
int32 curwords
Definition: ts_utils.h:106
ParsedWord * words
Definition: ts_utils.h:104
uint16 alen
Definition: ts_utils.h:87
uint16 flags
Definition: ts_utils.h:84
uint16 nvariant
Definition: ts_utils.h:86
uint16 len
Definition: ts_utils.h:85
uint16 pos
Definition: ts_utils.h:90
char * word
Definition: ts_utils.h:99
bool prefix
Definition: ts_type.h:163
uint32 distance
Definition: ts_type.h:172
uint32 length
Definition: ts_type.h:171
ListDictionary * map
Definition: ts_cache.h:80
FmgrInfo lexize
Definition: ts_cache.h:59
uint16 nvariant
Definition: ts_public.h:134
char * lexeme
Definition: ts_public.h:138
uint16 flags
Definition: ts_public.h:136
FmgrInfo prstoken
Definition: ts_cache.h:46
FmgrInfo prsstart
Definition: ts_cache.h:45
FmgrInfo prsend
Definition: ts_cache.h:47
int32 size
Definition: ts_type.h:221
Definition: c.h:692
TSConfigCacheEntry * lookup_ts_config_cache(Oid cfgId)
Definition: ts_cache.c:387
TSParserCacheEntry * lookup_ts_parser_cache(Oid prsId)
Definition: ts_cache.c:113
TSDictionaryCacheEntry * lookup_ts_dictionary_cache(Oid dictId)
Definition: ts_cache.c:208
static void hlfinditem(HeadlineParsedText *prs, TSQuery query, int32 pos, char *buf, int buflen)
Definition: ts_parse.c:464
static void setNewTmpRes(LexizeData *ld, ParsedLex *lex, TSLexeme *res)
Definition: ts_parse.c:158
text * generateHeadline(HeadlineParsedText *prs)
Definition: ts_parse.c:607
struct ListParsedLex ListParsedLex
static void RemoveHead(LexizeData *ld)
Definition: ts_parse.c:112
static void setCorrLex(LexizeData *ld, ParsedLex **correspondLexem)
Definition: ts_parse.c:120
void parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen)
Definition: ts_parse.c:355
void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int buflen)
Definition: ts_parse.c:540
static void hladdword(HeadlineParsedText *prs, char *buf, int buflen, int type)
Definition: ts_parse.c:440
static TSLexeme * LexizeExec(LexizeData *ld, ParsedLex **correspondLexem)
Definition: ts_parse.c:173
static void LPLAddTail(ListParsedLex *list, ParsedLex *newpl)
Definition: ts_parse.c:73
static void moveToWaste(LexizeData *ld, ParsedLex *stop)
Definition: ts_parse.c:142
struct ParsedLex ParsedLex
static ParsedLex * LPLRemoveHead(ListParsedLex *list)
Definition: ts_parse.c:86
static void addHLParsedLex(HeadlineParsedText *prs, TSQuery query, ParsedLex *lexs, TSLexeme *norms)
Definition: ts_parse.c:499
static void LexizeAddLemm(LexizeData *ld, int type, char *lemm, int lenlemm)
Definition: ts_parse.c:100
static void LexizeInit(LexizeData *ld, TSConfigCacheEntry *cfg)
Definition: ts_parse.c:61
#define TSL_PREFIX
Definition: ts_public.h:143
#define TSL_FILTER
Definition: ts_public.h:144
#define TSL_ADDPOS
Definition: ts_public.h:142
#define QI_VAL
Definition: ts_type.h:149
#define MAXSTRLEN
Definition: ts_type.h:49
#define LIMITPOS(x)
Definition: ts_type.h:87
int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix)
Definition: tsvector_op.c:1152
QueryOperand qoperand
Definition: ts_type.h:210
QueryItemType type
Definition: ts_type.h:208
static void SET_VARSIZE(void *PTR, Size len)
Definition: varatt.h:432
const char * type

AltStyle によって変換されたページ (->オリジナル) /