1/*-------------------------------------------------------------------------
4 * Retrieve compressed or external variable size attributes.
6 * Copyright (c) 2000-2025, PostgreSQL Global Development Group
9 * src/backend/access/common/detoast.c
11 *-------------------------------------------------------------------------
33 * detoast_external_attr -
35 * Public entry point to get back a toasted value from
36 * external source (possibly still in compressed format).
38 * This will return a datum that contains all the data internally, ie, not
39 * relying on external storage or memory, but it can still be compressed or
40 * have a short header. Note some callers assume that if the input is an
41 * EXTERNAL datum, the result will be a pfree'able chunk.
52 * This is an external stored plain value
59 * This is an indirect pointer --- dereference it
66 /* nested indirect Datums aren't allowed */
69 /* recurse if value is still external in some other way */
74 * Copy into the caller's memory context, in case caller tries to
83 * This is an expanded-object pointer --- get flat format
96 * This is a plain value inside of the main tuple - why am I called?
108 * Public entry point to get back a toasted value from compression
109 * or external storage. The result is always non-extended varlena form.
111 * Note some callers assume that if the input is an EXTERNAL or COMPRESSED
112 * datum, the result will be a pfree'able chunk.
121 * This is an externally stored datum --- fetch it back from there
124 /* If it's compressed, decompress it */
136 * This is an indirect pointer --- dereference it
143 /* nested indirect Datums aren't allowed */
146 /* recurse in case value is still extended in some other way */
149 /* if it isn't, we'd better copy it */
162 * This is an expanded-object pointer --- get flat format
165 /* flatteners are not allowed to produce compressed/short output */
171 * This is a compressed value inside of the main tuple
178 * This is a short-header varlena --- convert to 4-byte header format
195 * detoast_attr_slice -
197 * Public entry point to get back part of a toasted value
198 * from compression or external storage.
200 * sliceoffset is where to start (zero or more)
201 * If slicelength < 0, return everything beyond sliceoffset
215 elog(
ERROR,
"invalid sliceoffset: %d", sliceoffset);
218 * Compute slicelimit = offset + length, or -1 if we must fetch all of the
219 * value. In case of integer overflow, we must fetch all.
224 slicelength = slicelimit = -1;
232 /* fast path for non-compressed external datums */
237 * For compressed values, we need to fetch enough slices to decompress
238 * at least the requested part (when a prefix is requested).
239 * Otherwise, just fetch all slices.
246 * Determine maximum amount of compressed data needed for a prefix
247 * of a given length (after decompression).
249 * At least for now, if it's LZ4 data, we'll have to fetch the
250 * whole thing, because there doesn't seem to be an API call to
251 * determine how much compressed data we need to be sure of being
252 * able to decompress the required slice.
259 * Fetch enough compressed slices (compressed marker will get set
273 /* nested indirect Datums aren't allowed */
277 sliceoffset, slicelength);
281 /* pass it off to detoast_external_attr to flatten */
291 struct varlena *tmp = preslice;
293 /* Decompress enough to encompass the slice and the offset */
314 /* slicing of datum for compressed cases and plain value */
316 if (sliceoffset >= attrsize)
321 else if (slicelength < 0 || slicelimit > attrsize)
322 slicelength = attrsize - sliceoffset;
327 memcpy(
VARDATA(result), attrdata + sliceoffset, slicelength);
329 if (preslice != attr)
336 * toast_fetch_datum -
338 * Reconstruct an in memory Datum from the chunks saved
339 * in the toast relation
351 elog(
ERROR,
"toast_fetch_datum shouldn't be called for non-ondisk datums");
353 /* Must copy to access aligned fields */
366 return result;
/* Probably shouldn't happen, but just in
370 * Open the toast relation and its indexes
374 /* Fetch all chunks */
376 attrsize, 0, attrsize, result);
378 /* Close toast table */
385 * toast_fetch_datum_slice -
387 * Reconstruct a segment of a Datum from the chunks saved
388 * in the toast relation
390 * Note that this function supports non-compressed external datums
391 * and compressed external datums (in which case the requested slice
392 * has to be a prefix, i.e. sliceoffset has to be 0).
405 elog(
ERROR,
"toast_fetch_datum_slice shouldn't be called for non-ondisk datums");
407 /* Must copy to access aligned fields */
411 * It's nonsense to fetch slices of a compressed datum unless when it's a
412 * prefix -- this isn't lo_* we can't return a compressed datum which is
413 * meaningful to toast later.
419 if (sliceoffset >= attrsize)
426 * When fetching a prefix of a compressed external datum, account for the
427 * space required by va_tcinfo, which is stored at the beginning as an
431 slicelength = slicelength +
sizeof(
int32);
434 * Adjust length request if needed. (Note: our sole caller,
435 * detoast_attr_slice, protects us against sliceoffset + slicelength
438 if (((sliceoffset + slicelength) > attrsize) || slicelength < 0)
439 slicelength = attrsize - sliceoffset;
448 if (slicelength == 0)
449 return result;
/* Can save a lot of work at this point! */
451 /* Open the toast relation */
454 /* Fetch all chunks */
456 attrsize, sliceoffset, slicelength,
459 /* Close toast table */
466 * toast_decompress_datum -
468 * Decompress a compressed version of a varlena datum
478 * Fetch the compression method id stored in the compression header and
479 * decompress the data using the appropriate decompression routine.
489 elog(
ERROR,
"invalid compression method id %d", cmid);
490 return NULL;
/* keep compiler quiet */
496 * toast_decompress_datum_slice -
498 * Decompress the front of a compressed version of a varlena datum.
499 * offset handling happens in detoast_attr_slice.
500 * Here we just decompress a slice from the front.
510 * Some callers may pass a slicelength that's more than the actual
511 * decompressed size. If so, just decompress normally. This avoids
512 * possibly allocating a larger-than-necessary result object, and may be
513 * faster and/or more robust as well. Notably, some versions of liblz4
514 * have been seen to give wrong results if passed an output size that is
515 * more than the data's true decompressed size.
521 * Fetch the compression method id stored in the compression header and
522 * decompress the data slice using the appropriate decompression routine.
532 elog(
ERROR,
"invalid compression method id %d", cmid);
533 return NULL;
/* keep compiler quiet */
538 * toast_raw_datum_size -
540 * Return the raw (detoasted) size of a varlena datum
541 * (including the VARHDRSZ header)
552 /* va_rawsize is the size of the original datum -- including header */
564 /* nested indirect Datums aren't allowed */
575 /* here, va_rawsize is just the payload size */
581 * we have to normalize the header length to VARHDRSZ or else the
582 * callers of this function will be confused.
588 /* plain untoasted datum */
597 * Return the physical storage size (possibly compressed) of a varlena datum
609 * Attribute is stored externally - return the extsize whether
610 * compressed or not. We do not count the size of the toast pointer
624 /* nested indirect Datums aren't allowed */
640 * Attribute is stored inline either compressed or not, just calculate
641 * the size of the datum in either case.
static struct varlena * toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
Size toast_datum_size(Datum value)
static struct varlena * toast_fetch_datum(struct varlena *attr)
struct varlena * detoast_attr(struct varlena *attr)
static struct varlena * toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength)
Size toast_raw_datum_size(Datum value)
struct varlena * detoast_external_attr(struct varlena *attr)
static struct varlena * toast_decompress_datum(struct varlena *attr)
struct varlena * detoast_attr_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength)
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
ExpandedObjectHeader * DatumGetEOHP(Datum d)
void EOH_flatten_into(ExpandedObjectHeader *eohptr, void *result, Size allocated_size)
Size EOH_get_flat_size(ExpandedObjectHeader *eohptr)
Assert(PointerIsAligned(start, uint64))
static bool pg_add_s32_overflow(int32 a, int32 b, int32 *result)
void pfree(void *pointer)
int32 pglz_maximum_compressed_size(int32 rawsize, int32 total_compressed_size)
static Datum PointerGetDatum(const void *X)
static Pointer DatumGetPointer(Datum X)
void table_close(Relation relation, LOCKMODE lockmode)
Relation table_open(Oid relationId, LOCKMODE lockmode)
static void table_relation_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result)
struct varlena * pglz_decompress_datum(const struct varlena *value)
struct varlena * pglz_decompress_datum_slice(const struct varlena *value, int32 slicelength)
struct varlena * lz4_decompress_datum(const struct varlena *value)
struct varlena * lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
@ TOAST_LZ4_COMPRESSION_ID
@ TOAST_PGLZ_COMPRESSION_ID
#define TOAST_COMPRESS_EXTSIZE(ptr)
#define TOAST_COMPRESS_METHOD(ptr)
static bool VARATT_IS_SHORT(const void *PTR)
static Size VARDATA_COMPRESSED_GET_EXTSIZE(const void *PTR)
static void SET_VARSIZE_COMPRESSED(void *PTR, Size len)
static bool VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
static Size VARATT_EXTERNAL_GET_EXTSIZE(struct varatt_external toast_pointer)
static Size VARSIZE_ANY(const void *PTR)
static bool VARATT_IS_EXTENDED(const void *PTR)
static bool VARATT_IS_EXTERNAL(const void *PTR)
static bool VARATT_IS_EXTERNAL_INDIRECT(const void *PTR)
static Size VARSIZE(const void *PTR)
static char * VARDATA(const void *PTR)
static bool VARATT_IS_COMPRESSED(const void *PTR)
static bool VARATT_IS_EXTERNAL_EXPANDED(const void *PTR)
static char * VARDATA_SHORT(const void *PTR)
static uint32 VARATT_EXTERNAL_GET_COMPRESS_METHOD(struct varatt_external toast_pointer)
static bool VARATT_EXTERNAL_IS_COMPRESSED(struct varatt_external toast_pointer)
static void SET_VARSIZE(void *PTR, Size len)
static Size VARSIZE_SHORT(const void *PTR)