PostgreSQL Source Code: src/backend/utils/adt/tsvector.c Source File

PostgreSQL Source Code git master
tsvector.c
Go to the documentation of this file.
1/*-------------------------------------------------------------------------
2 *
3 * tsvector.c
4 * I/O functions for tsvector
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/utils/adt/tsvector.c
11 *
12 *-------------------------------------------------------------------------
13 */
14
15#include "postgres.h"
16
17#include "common/int.h"
18#include "libpq/pqformat.h"
19#include "nodes/miscnodes.h"
20#include "tsearch/ts_locale.h"
21#include "tsearch/ts_utils.h"
22#include "utils/fmgrprotos.h"
23#include "utils/memutils.h"
24#include "varatt.h"
25
26 typedef struct
27{
28 WordEntry entry; /* must be first, see compareentry */
29 WordEntryPos *pos;
30 int poslen; /* number of elements in pos */
31} WordEntryIN;
32
33
34/* Compare two WordEntryPos values for qsort */
35int
36 compareWordEntryPos(const void *a, const void *b)
37{
38 int apos = WEP_GETPOS(*(const WordEntryPos *) a);
39 int bpos = WEP_GETPOS(*(const WordEntryPos *) b);
40
41 return pg_cmp_s32(apos, bpos);
42}
43
44/*
45 * Removes duplicate pos entries. If there's two entries with same pos but
46 * different weight, the higher weight is retained, so we can't use
47 * qunique here.
48 *
49 * Returns new length.
50 */
51static int
52 uniquePos(WordEntryPos *a, int l)
53{
54 WordEntryPos *ptr,
55 *res;
56
57 if (l <= 1)
58 return l;
59
60 qsort(a, l, sizeof(WordEntryPos), compareWordEntryPos);
61
62 res = a;
63 ptr = a + 1;
64 while (ptr - a < l)
65 {
66 if (WEP_GETPOS(*ptr) != WEP_GETPOS(*res))
67 {
68 res++;
69 *res = *ptr;
70 if (res - a >= MAXNUMPOS - 1 ||
71 WEP_GETPOS(*res) == MAXENTRYPOS - 1)
72 break;
73 }
74 else if (WEP_GETWEIGHT(*ptr) > WEP_GETWEIGHT(*res))
75 WEP_SETWEIGHT(*res, WEP_GETWEIGHT(*ptr));
76 ptr++;
77 }
78
79 return res + 1 - a;
80}
81
82/*
83 * Compare two WordEntry structs for qsort_arg. This can also be used on
84 * WordEntryIN structs, since those have WordEntry as their first field.
85 */
86static int
87 compareentry(const void *va, const void *vb, void *arg)
88{
89 const WordEntry *a = (const WordEntry *) va;
90 const WordEntry *b = (const WordEntry *) vb;
91 char *BufferStr = (char *) arg;
92
93 return tsCompareString(&BufferStr[a->pos], a->len,
94 &BufferStr[b->pos], b->len,
95 false);
96}
97
98/*
99 * Sort an array of WordEntryIN, remove duplicates.
100 * *outbuflen receives the amount of space needed for strings and positions.
101 */
102static int
103 uniqueentry(WordEntryIN *a, int l, char *buf, int *outbuflen)
104{
105 int buflen;
106 WordEntryIN *ptr,
107 *res;
108
109 Assert(l >= 1);
110
111 if (l > 1)
112 qsort_arg(a, l, sizeof(WordEntryIN), compareentry, buf);
113
114 buflen = 0;
115 res = a;
116 ptr = a + 1;
117 while (ptr - a < l)
118 {
119 if (!(ptr->entry.len == res->entry.len &&
120 strncmp(&buf[ptr->entry.pos], &buf[res->entry.pos],
121 res->entry.len) == 0))
122 {
123 /* done accumulating data into *res, count space needed */
124 buflen += res->entry.len;
125 if (res->entry.haspos)
126 {
127 res->poslen = uniquePos(res->pos, res->poslen);
128 buflen = SHORTALIGN(buflen);
129 buflen += res->poslen * sizeof(WordEntryPos) + sizeof(uint16);
130 }
131 res++;
132 if (res != ptr)
133 memcpy(res, ptr, sizeof(WordEntryIN));
134 }
135 else if (ptr->entry.haspos)
136 {
137 if (res->entry.haspos)
138 {
139 /* append ptr's positions to res's positions */
140 int newlen = ptr->poslen + res->poslen;
141
142 res->pos = (WordEntryPos *)
143 repalloc(res->pos, newlen * sizeof(WordEntryPos));
144 memcpy(&res->pos[res->poslen], ptr->pos,
145 ptr->poslen * sizeof(WordEntryPos));
146 res->poslen = newlen;
147 pfree(ptr->pos);
148 }
149 else
150 {
151 /* just give ptr's positions to pos */
152 res->entry.haspos = 1;
153 res->pos = ptr->pos;
154 res->poslen = ptr->poslen;
155 }
156 }
157 ptr++;
158 }
159
160 /* count space needed for last item */
161 buflen += res->entry.len;
162 if (res->entry.haspos)
163 {
164 res->poslen = uniquePos(res->pos, res->poslen);
165 buflen = SHORTALIGN(buflen);
166 buflen += res->poslen * sizeof(WordEntryPos) + sizeof(uint16);
167 }
168
169 *outbuflen = buflen;
170 return res + 1 - a;
171}
172
173
174Datum
175 tsvectorin(PG_FUNCTION_ARGS)
176{
177 char *buf = PG_GETARG_CSTRING(0);
178 Node *escontext = fcinfo->context;
179 TSVectorParseState state;
180 WordEntryIN *arr;
181 int totallen;
182 int arrlen; /* allocated size of arr */
183 WordEntry *inarr;
184 int len = 0;
185 TSVector in;
186 int i;
187 char *token;
188 int toklen;
189 WordEntryPos *pos;
190 int poslen;
191 char *strbuf;
192 int stroff;
193
194 /*
195 * Tokens are appended to tmpbuf, cur is a pointer to the end of used
196 * space in tmpbuf.
197 */
198 char *tmpbuf;
199 char *cur;
200 int buflen = 256; /* allocated size of tmpbuf */
201
202 state = init_tsvector_parser(buf, 0, escontext);
203
204 arrlen = 64;
205 arr = (WordEntryIN *) palloc(sizeof(WordEntryIN) * arrlen);
206 cur = tmpbuf = (char *) palloc(buflen);
207
208 while (gettoken_tsvector(state, &token, &toklen, &pos, &poslen, NULL))
209 {
210 if (toklen >= MAXSTRLEN)
211 ereturn(escontext, (Datum) 0,
212 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
213 errmsg("word is too long (%ld bytes, max %ld bytes)",
214 (long) toklen,
215 (long) (MAXSTRLEN - 1))));
216
217 if (cur - tmpbuf > MAXSTRPOS)
218 ereturn(escontext, (Datum) 0,
219 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
220 errmsg("string is too long for tsvector (%ld bytes, max %ld bytes)",
221 (long) (cur - tmpbuf), (long) MAXSTRPOS)));
222
223 /*
224 * Enlarge buffers if needed
225 */
226 if (len >= arrlen)
227 {
228 arrlen *= 2;
229 arr = (WordEntryIN *)
230 repalloc(arr, sizeof(WordEntryIN) * arrlen);
231 }
232 while ((cur - tmpbuf) + toklen >= buflen)
233 {
234 int dist = cur - tmpbuf;
235
236 buflen *= 2;
237 tmpbuf = (char *) repalloc(tmpbuf, buflen);
238 cur = tmpbuf + dist;
239 }
240 arr[len].entry.len = toklen;
241 arr[len].entry.pos = cur - tmpbuf;
242 memcpy(cur, token, toklen);
243 cur += toklen;
244
245 if (poslen != 0)
246 {
247 arr[len].entry.haspos = 1;
248 arr[len].pos = pos;
249 arr[len].poslen = poslen;
250 }
251 else
252 {
253 arr[len].entry.haspos = 0;
254 arr[len].pos = NULL;
255 arr[len].poslen = 0;
256 }
257 len++;
258 }
259
260 close_tsvector_parser(state);
261
262 /* Did gettoken_tsvector fail? */
263 if (SOFT_ERROR_OCCURRED(escontext))
264 PG_RETURN_NULL();
265
266 if (len > 0)
267 len = uniqueentry(arr, len, tmpbuf, &buflen);
268 else
269 buflen = 0;
270
271 if (buflen > MAXSTRPOS)
272 ereturn(escontext, (Datum) 0,
273 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
274 errmsg("string is too long for tsvector (%d bytes, max %d bytes)", buflen, MAXSTRPOS)));
275
276 totallen = CALCDATASIZE(len, buflen);
277 in = (TSVector) palloc0(totallen);
278 SET_VARSIZE(in, totallen);
279 in->size = len;
280 inarr = ARRPTR(in);
281 strbuf = STRPTR(in);
282 stroff = 0;
283 for (i = 0; i < len; i++)
284 {
285 memcpy(strbuf + stroff, &tmpbuf[arr[i].entry.pos], arr[i].entry.len);
286 arr[i].entry.pos = stroff;
287 stroff += arr[i].entry.len;
288 if (arr[i].entry.haspos)
289 {
290 /* This should be unreachable because of MAXNUMPOS restrictions */
291 if (arr[i].poslen > 0xFFFF)
292 elog(ERROR, "positions array too long");
293
294 /* Copy number of positions */
295 stroff = SHORTALIGN(stroff);
296 *(uint16 *) (strbuf + stroff) = (uint16) arr[i].poslen;
297 stroff += sizeof(uint16);
298
299 /* Copy positions */
300 memcpy(strbuf + stroff, arr[i].pos, arr[i].poslen * sizeof(WordEntryPos));
301 stroff += arr[i].poslen * sizeof(WordEntryPos);
302
303 pfree(arr[i].pos);
304 }
305 inarr[i] = arr[i].entry;
306 }
307
308 Assert((strbuf + stroff - (char *) in) == totallen);
309
310 PG_RETURN_TSVECTOR(in);
311}
312
313Datum
314 tsvectorout(PG_FUNCTION_ARGS)
315{
316 TSVector out = PG_GETARG_TSVECTOR(0);
317 char *outbuf;
318 int32 i,
319 lenbuf = 0,
320 pp;
321 WordEntry *ptr = ARRPTR(out);
322 char *curbegin,
323 *curin,
324 *curout;
325
326 lenbuf = out->size * 2 /* '' */ + out->size - 1 /* space */ + 2 /* 0円 */ ;
327 for (i = 0; i < out->size; i++)
328 {
329 lenbuf += ptr[i].len * 2 * pg_database_encoding_max_length() /* for escape */ ;
330 if (ptr[i].haspos)
331 lenbuf += 1 /* : */ + 7 /* int2 + , + weight */ * POSDATALEN(out, &(ptr[i]));
332 }
333
334 curout = outbuf = (char *) palloc(lenbuf);
335 for (i = 0; i < out->size; i++)
336 {
337 curbegin = curin = STRPTR(out) + ptr->pos;
338 if (i != 0)
339 *curout++ = ' ';
340 *curout++ = '\'';
341 while (curin - curbegin < ptr->len)
342 {
343 int len = pg_mblen(curin);
344
345 if (t_iseq(curin, '\''))
346 *curout++ = '\'';
347 else if (t_iseq(curin, '\\'))
348 *curout++ = '\\';
349
350 while (len--)
351 *curout++ = *curin++;
352 }
353
354 *curout++ = '\'';
355 if ((pp = POSDATALEN(out, ptr)) != 0)
356 {
357 WordEntryPos *wptr;
358
359 *curout++ = ':';
360 wptr = POSDATAPTR(out, ptr);
361 while (pp)
362 {
363 curout += sprintf(curout, "%d", WEP_GETPOS(*wptr));
364 switch (WEP_GETWEIGHT(*wptr))
365 {
366 case 3:
367 *curout++ = 'A';
368 break;
369 case 2:
370 *curout++ = 'B';
371 break;
372 case 1:
373 *curout++ = 'C';
374 break;
375 case 0:
376 default:
377 break;
378 }
379
380 if (pp > 1)
381 *curout++ = ',';
382 pp--;
383 wptr++;
384 }
385 }
386 ptr++;
387 }
388
389 *curout = '0円';
390 PG_FREE_IF_COPY(out, 0);
391 PG_RETURN_CSTRING(outbuf);
392}
393
394/*
395 * Binary Input / Output functions. The binary format is as follows:
396 *
397 * uint32 number of lexemes
398 *
399 * for each lexeme:
400 * lexeme text in client encoding, null-terminated
401 * uint16 number of positions
402 * for each position:
403 * uint16 WordEntryPos
404 */
405
406Datum
407 tsvectorsend(PG_FUNCTION_ARGS)
408{
409 TSVector vec = PG_GETARG_TSVECTOR(0);
410 StringInfoData buf;
411 int i,
412 j;
413 WordEntry *weptr = ARRPTR(vec);
414
415 pq_begintypsend(&buf);
416
417 pq_sendint32(&buf, vec->size);
418 for (i = 0; i < vec->size; i++)
419 {
420 uint16 npos;
421
422 /*
423 * the strings in the TSVector array are not null-terminated, so we
424 * have to send the null-terminator separately
425 */
426 pq_sendtext(&buf, STRPTR(vec) + weptr->pos, weptr->len);
427 pq_sendbyte(&buf, '0円');
428
429 npos = POSDATALEN(vec, weptr);
430 pq_sendint16(&buf, npos);
431
432 if (npos > 0)
433 {
434 WordEntryPos *wepptr = POSDATAPTR(vec, weptr);
435
436 for (j = 0; j < npos; j++)
437 pq_sendint16(&buf, wepptr[j]);
438 }
439 weptr++;
440 }
441
442 PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
443}
444
445Datum
446 tsvectorrecv(PG_FUNCTION_ARGS)
447{
448 StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
449 TSVector vec;
450 int i;
451 int32 nentries;
452 int datalen; /* number of bytes used in the variable size
453 * area after fixed size TSVector header and
454 * WordEntries */
455 Size hdrlen;
456 Size len; /* allocated size of vec */
457 bool needSort = false;
458
459 nentries = pq_getmsgint(buf, sizeof(int32));
460 if (nentries < 0 || nentries > (MaxAllocSize / sizeof(WordEntry)))
461 elog(ERROR, "invalid size of tsvector");
462
463 hdrlen = DATAHDRSIZE + sizeof(WordEntry) * nentries;
464
465 len = hdrlen * 2; /* times two to make room for lexemes */
466 vec = (TSVector) palloc0(len);
467 vec->size = nentries;
468
469 datalen = 0;
470 for (i = 0; i < nentries; i++)
471 {
472 const char *lexeme;
473 uint16 npos;
474 size_t lex_len;
475
476 lexeme = pq_getmsgstring(buf);
477 npos = (uint16) pq_getmsgint(buf, sizeof(uint16));
478
479 /* sanity checks */
480
481 lex_len = strlen(lexeme);
482 if (lex_len > MAXSTRLEN)
483 elog(ERROR, "invalid tsvector: lexeme too long");
484
485 if (datalen > MAXSTRPOS)
486 elog(ERROR, "invalid tsvector: maximum total lexeme length exceeded");
487
488 if (npos > MAXNUMPOS)
489 elog(ERROR, "unexpected number of tsvector positions");
490
491 /*
492 * Looks valid. Fill the WordEntry struct, and copy lexeme.
493 *
494 * But make sure the buffer is large enough first.
495 */
496 while (hdrlen + SHORTALIGN(datalen + lex_len) +
497 sizeof(uint16) + npos * sizeof(WordEntryPos) >= len)
498 {
499 len *= 2;
500 vec = (TSVector) repalloc(vec, len);
501 }
502
503 vec->entries[i].haspos = (npos > 0) ? 1 : 0;
504 vec->entries[i].len = lex_len;
505 vec->entries[i].pos = datalen;
506
507 memcpy(STRPTR(vec) + datalen, lexeme, lex_len);
508
509 datalen += lex_len;
510
511 if (i > 0 && compareentry(&vec->entries[i],
512 &vec->entries[i - 1],
513 STRPTR(vec)) <= 0)
514 needSort = true;
515
516 /* Receive positions */
517 if (npos > 0)
518 {
519 uint16 j;
520 WordEntryPos *wepptr;
521
522 /*
523 * Pad to 2-byte alignment if necessary. Though we used palloc0
524 * for the initial allocation, subsequent repalloc'd memory areas
525 * are not initialized to zero.
526 */
527 if (datalen != SHORTALIGN(datalen))
528 {
529 *(STRPTR(vec) + datalen) = '0円';
530 datalen = SHORTALIGN(datalen);
531 }
532
533 memcpy(STRPTR(vec) + datalen, &npos, sizeof(uint16));
534
535 wepptr = POSDATAPTR(vec, &vec->entries[i]);
536 for (j = 0; j < npos; j++)
537 {
538 wepptr[j] = (WordEntryPos) pq_getmsgint(buf, sizeof(WordEntryPos));
539 if (j > 0 && WEP_GETPOS(wepptr[j]) <= WEP_GETPOS(wepptr[j - 1]))
540 elog(ERROR, "position information is misordered");
541 }
542
543 datalen += sizeof(uint16) + npos * sizeof(WordEntryPos);
544 }
545 }
546
547 SET_VARSIZE(vec, hdrlen + datalen);
548
549 if (needSort)
550 qsort_arg(ARRPTR(vec), vec->size, sizeof(WordEntry),
551 compareentry, STRPTR(vec));
552
553 PG_RETURN_TSVECTOR(vec);
554}
#define SHORTALIGN(LEN)
Definition: c.h:806
int32_t int32
Definition: c.h:534
uint16_t uint16
Definition: c.h:537
size_t Size
Definition: c.h:610
#define ARRPTR(x)
Definition: cube.c:28
struct cursor * cur
Definition: ecpg.c:29
int errcode(int sqlerrcode)
Definition: elog.c:854
int errmsg(const char *fmt,...)
Definition: elog.c:1071
#define ereturn(context, dummy_value,...)
Definition: elog.h:278
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:226
#define MaxAllocSize
Definition: fe_memutils.h:22
#define PG_FREE_IF_COPY(ptr, n)
Definition: fmgr.h:260
#define PG_RETURN_BYTEA_P(x)
Definition: fmgr.h:371
#define PG_GETARG_POINTER(n)
Definition: fmgr.h:276
#define PG_RETURN_CSTRING(x)
Definition: fmgr.h:362
#define PG_GETARG_CSTRING(n)
Definition: fmgr.h:277
#define PG_RETURN_NULL()
Definition: fmgr.h:345
#define PG_FUNCTION_ARGS
Definition: fmgr.h:193
Assert(PointerIsAligned(start, uint64))
#define CALCDATASIZE(x, lenstr)
Definition: hstore.h:72
#define STRPTR(x)
Definition: hstore.h:76
#define token
Definition: indent_globs.h:126
static int pg_cmp_s32(int32 a, int32 b)
Definition: int.h:646
b
int b
Definition: isn.c:74
a
int a
Definition: isn.c:73
j
int j
Definition: isn.c:78
i
int i
Definition: isn.c:77
int pg_database_encoding_max_length(void)
Definition: mbutils.c:1547
int pg_mblen(const char *mbstr)
Definition: mbutils.c:1024
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1610
void pfree(void *pointer)
Definition: mcxt.c:1594
void * palloc0(Size size)
Definition: mcxt.c:1395
void * palloc(Size size)
Definition: mcxt.c:1365
#define SOFT_ERROR_OCCURRED(escontext)
Definition: miscnodes.h:53
void * arg
const void size_t len
static char * buf
Definition: pg_test_fsync.c:72
#define sprintf
Definition: port.h:241
void qsort_arg(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
#define qsort(a, b, c, d)
Definition: port.h:479
uint64_t Datum
Definition: postgres.h:70
unsigned int pq_getmsgint(StringInfo msg, int b)
Definition: pqformat.c:415
void pq_sendtext(StringInfo buf, const char *str, int slen)
Definition: pqformat.c:172
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:579
void pq_begintypsend(StringInfo buf)
Definition: pqformat.c:326
bytea * pq_endtypsend(StringInfo buf)
Definition: pqformat.c:346
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:144
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:160
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:136
struct StringInfoData * StringInfo
Definition: string.h:15
Definition: nodes.h:135
int32 size
Definition: ts_type.h:93
WordEntry entries[FLEXIBLE_ARRAY_MEMBER]
Definition: ts_type.h:94
int poslen
Definition: tsvector.c:30
WordEntryPos * pos
Definition: tsvector.c:29
WordEntry entry
Definition: tsvector.c:28
Definition: ts_type.h:42
uint32 pos
Definition: ts_type.h:46
uint32 haspos
Definition: ts_type.h:44
uint32 len
Definition: ts_type.h:45
Definition: regguts.h:323
Definition: oauth-curl.c:192
#define t_iseq(x, c)
Definition: ts_locale.h:38
#define PG_GETARG_TSVECTOR(n)
Definition: ts_type.h:135
#define WEP_GETPOS(x)
Definition: ts_type.h:80
#define PG_RETURN_TSVECTOR(x)
Definition: ts_type.h:137
#define MAXENTRYPOS
Definition: ts_type.h:85
#define POSDATALEN(x, e)
Definition: ts_type.h:110
uint16 WordEntryPos
Definition: ts_type.h:63
#define MAXNUMPOS
Definition: ts_type.h:86
TSVectorData * TSVector
Definition: ts_type.h:98
#define WEP_SETWEIGHT(x, v)
Definition: ts_type.h:82
#define DATAHDRSIZE
Definition: ts_type.h:100
#define MAXSTRLEN
Definition: ts_type.h:49
#define POSDATAPTR(x, e)
Definition: ts_type.h:111
#define WEP_GETWEIGHT(x)
Definition: ts_type.h:79
#define MAXSTRPOS
Definition: ts_type.h:50
Datum tsvectorout(PG_FUNCTION_ARGS)
Definition: tsvector.c:314
static int uniquePos(WordEntryPos *a, int l)
Definition: tsvector.c:52
Datum tsvectorrecv(PG_FUNCTION_ARGS)
Definition: tsvector.c:446
static int compareentry(const void *va, const void *vb, void *arg)
Definition: tsvector.c:87
Datum tsvectorin(PG_FUNCTION_ARGS)
Definition: tsvector.c:175
int compareWordEntryPos(const void *a, const void *b)
Definition: tsvector.c:36
Datum tsvectorsend(PG_FUNCTION_ARGS)
Definition: tsvector.c:407
static int uniqueentry(WordEntryIN *a, int l, char *buf, int *outbuflen)
Definition: tsvector.c:103
int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix)
Definition: tsvector_op.c:1152
void close_tsvector_parser(TSVectorParseState state)
bool gettoken_tsvector(TSVectorParseState state, char **strval, int *lenval, WordEntryPos **pos_ptr, int *poslen, char **endptr)
TSVectorParseState init_tsvector_parser(char *input, int flags, Node *escontext)
static void SET_VARSIZE(void *PTR, Size len)
Definition: varatt.h:432
static StringInfoData tmpbuf
Definition: walsender.c:178

AltStyle によって変換されたページ (->オリジナル) /