1/*-------------------------------------------------------------------------
4 * Functions for internal use by the TOAST system.
6 * Copyright (c) 2000-2025, PostgreSQL Global Development Group
9 * src/backend/access/common/toast_internals.c
11 *-------------------------------------------------------------------------
25#include "utils/fmgroids.h"
33 * toast_compress_datum -
35 * Create a compressed version of a varlena datum
37 * If we fail (ie, compressed result is actually bigger than original)
38 * then return NULL. We must not use compressed data if it'd expand
41 * We use VAR{SIZE,DATA}_ANY so we can handle short varlenas here without
42 * copying them. But we can't handle external or compressed datums.
57 /* If the compression method is not valid, use the current default */
62 * Call appropriate compression routine for the compression method.
75 elog(
ERROR,
"invalid compression method %c", cmethod);
82 * We recheck the actual size even if compression reports success, because
83 * it might be satisfied with having saved as little as one byte in the
84 * compressed data --- which could turn into a net loss once you consider
85 * header and alignment padding. Worst case, the compressed format might
86 * require three padding bytes (plus header, which is included in
87 * VARSIZE(tmp)), whereas the uncompressed format would take only one
88 * header byte and no padding if the value is short enough. So we insist
89 * on a savings of more than 2 bytes to ensure we have a gain.
93 /* successful compression */
100 /* incompressible data */
109 * Save one single datum into the secondary relation and return
110 * a Datum reference for it.
112 * rel: the main relation we're working with (not the toast rel!)
113 * value: datum to be pushed to toast storage
114 * oldexternal: if not NULL, toast pointer previously representing the datum
115 * options: options to be passed to heap_insert() for toast rows
138 * Open the toast relation and its indexes. We can use the index to check
139 * uniqueness of the OID we assign to the toasted item, even though it has
140 * additional columns besides OID.
143 toasttupDesc = toastrel->
rd_att;
145 /* Open all the toast indexes and look for the valid one */
152 * Get the data pointer and length, and compute va_rawsize and va_extinfo.
154 * va_rawsize is the size of the equivalent fully uncompressed datum, so
155 * we have to adjust for short headers.
157 * va_extinfo stored the actual size of the data payload in the toast
158 * records and the compression method in first 2 bits if data is
172 /* rawsize in a compressed datum is just the size of the payload */
175 /* set external size and compression method */
178 /* Assert that the numbers look like it's compressed */
190 * Insert the correct table OID into the result TOAST pointer.
192 * Normally this is the actual OID of the target toast table, but during
193 * table-rewriting operations such as CLUSTER, we have to insert the OID
194 * of the table's real permanent toast table instead. rd_toastoid is set
195 * if we have to substitute such an OID.
203 * Choose an OID to use as the value ID for this toast value.
205 * Normally we just choose an unused OID within the toast table. But
206 * during table-rewriting operations where we are preserving an existing
207 * toast table OID, we want to preserve toast value OIDs too. So, if
208 * rd_toastoid is set and we had a prior external value from that same
209 * toast table, re-use its value ID. If we didn't have a prior external
210 * value (which is a corner case, but possible if the table's attstorage
211 * options have been changed), we have to pick a value ID that doesn't
212 * conflict with either new or existing toast value OIDs.
216 /* normal case: just choose an unused OID */
224 /* rewrite case: check to see if value was in old toast table */
226 if (oldexternal != NULL)
231 /* Must copy to access aligned fields */
235 /* This value came from the old toast table; reuse its OID */
239 * There is a corner case here: the table rewrite might have
240 * to copy both live and recently-dead versions of a row, and
241 * those versions could easily reference the same toast value.
242 * When we copy the second or later version of such a row,
243 * reusing the OID will mean we select an OID that's already
244 * in the new toast table. Check for that, and if so, just
245 * fall through without writing the data again.
247 * While annoying and ugly-looking, this is a good thing
248 * because it ensures that we wind up with only one copy of
249 * the toast value when there is only one copy in the old
250 * toast table. Before we detected this case, we'd have made
251 * multiple copies, wasting space; and what's worse, the
252 * copies belonging to already-deleted heap tuples would not
253 * be reclaimed by VACUUM.
258 /* Match, so short-circuit the data storage loop below */
266 * new value; must choose an OID that doesn't conflict in either
267 * old or new toast table
281 * Split up the item into chunks
283 while (data_todo > 0)
287 bool t_isnull[3] = {0};
291 /* this is to make the union big enough for a chunk: */
293 /* ensure union is aligned well enough: */
301 * Calculate the size of this chunk
306 * Build a tuple and store it
311 memcpy(
VARDATA(&chunk_data), data_p, chunk_size);
319 * Create the index entry. We cheat a little here by not using
320 * FormIndexDatum: this relies on the knowledge that the index columns
321 * are the same as the initial columns of the table for all the
322 * indexes. We also cheat by not providing an IndexInfo: this is okay
323 * for now because btree doesn't need one, but we might have to be
324 * more honest someday.
326 * Note also that there had better not be any user-created index on
327 * the TOAST table, since we don't bother to update anything else.
329 for (
int i = 0;
i < num_indexes;
i++)
331 /* Only index relations marked as ready can be updated */
332 if (toastidxs[
i]->rd_index->indisready)
347 * Move on to next chunk
349 data_todo -= chunk_size;
350 data_p += chunk_size;
354 * Done - close toast relation and its indexes but keep the lock until
355 * commit, so as a concurrent reindex done directly on the toast relation
356 * would be able to wait for this transaction.
362 * Create the TOAST pointer value that we'll return
372 * toast_delete_datum -
374 * Delete a single external stored value.
393 /* Must copy to access aligned fields */
397 * Open the toast relation and its indexes
401 /* Fetch valid relation used for process */
408 * Setup a scan key to find chunks with matching va_valueid
416 * Find all the chunks. (We don't actually care whether we see them in
417 * sequence or not, but since we've already locked the index we might as
418 * well use systable_beginscan_ordered.)
425 * Have a chunk, delete it
434 * End scan and close relations but keep the lock until commit, so as a
435 * concurrent reindex done directly on the toast relation would be able to
436 * wait for this transaction.
444 * toastrel_valueid_exists -
446 * Test whether a toast value with the given ID exists in the toast relation.
447 * For safety, we consider a value to exist if there are either live or dead
448 * toast rows with that ID; see notes for GetNewOidWithIndex().
461 /* Fetch a valid index relation */
468 * Setup a scan key to find chunks with matching va_valueid
476 * Is there any such chunk?
494 * toastid_valueid_exists -
496 * As above, but work from toast rel's OID not an open relation
515 * toast_get_valid_index
517 * Get OID of valid index associated to given toast relation. A toast
518 * relation can have only one valid index at the same time.
529 /* Open the toast relation */
532 /* Look for the valid index of the toast relation */
539 /* Close the toast relation and all its indexes */
543 return validIndexOid;
549 * Get an array of the indexes associated to the given toast relation
550 * and return as well the position of the valid index used by the toast
551 * relation in this array. It is the responsibility of the caller of this
552 * function to close the indexes as well as free them.
566 /* Get index list of the toast relation */
572 /* Open all the index relations */
574 foreach(lc, indexlist)
577 /* Fetch the first valid index in list */
578 for (
i = 0;
i < *num_indexes;
i++)
591 * Free index list, not necessary anymore as relations are opened and a
592 * valid index has been found.
597 * The toast relation should have one valid index, so something is going
598 * wrong if there is nothing.
601 elog(
ERROR,
"no valid index found for toast relation with Oid %u",
608 * toast_close_indexes
610 * Close an array of indexes for a toast relation and free it. This should
611 * be called for a set of indexes opened previously with toast_open_indexes.
618 /* Close relations and clean up things */
619 for (
i = 0;
i < num_indexes;
i++)
627 * Return the TOAST snapshot. Detoasting *must* happen in the same
628 * transaction that originally fetched the toast pointer.
634 * We cannot directly check that detoasting happens in the same
635 * transaction that originally fetched the toast pointer, but at least
636 * check that the session has some active snapshots. It might not if, for
637 * example, a procedure fetches a toasted value into a local variable,
638 * commits, and then tries to detoast the value. Such coding is unsafe,
639 * because once we commit there is nothing to prevent the toast data from
640 * being deleted. (This is not very much protection, because in many
641 * scenarios the procedure would have already created a new transaction
642 * snapshot, preventing us from detecting the problem. But it's better
646 elog(
ERROR,
"cannot fetch toast data without an active snapshot");
#define OidIsValid(objectId)
Oid GetNewOidWithIndex(Relation relation, Oid indexId, AttrNumber oidcolumn)
#define TOAST_POINTER_SIZE
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
void systable_endscan(SysScanDesc sysscan)
SysScanDesc systable_beginscan_ordered(Relation heapRelation, Relation indexRelation, Snapshot snapshot, int nkeys, ScanKey key)
HeapTuple systable_getnext(SysScanDesc sysscan)
void systable_endscan_ordered(SysScanDesc sysscan)
HeapTuple systable_getnext_ordered(SysScanDesc sysscan, ScanDirection direction)
SysScanDesc systable_beginscan(Relation heapRelation, Oid indexId, bool indexOK, Snapshot snapshot, int nkeys, ScanKey key)
Assert(PointerIsAligned(start, uint64))
void heap_insert(Relation relation, HeapTuple tup, CommandId cid, int options, BulkInsertState bistate)
void simple_heap_delete(Relation relation, ItemPointer tid)
void heap_abort_speculative(Relation relation, ItemPointer tid)
#define TOAST_MAX_CHUNK_SIZE
HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, const Datum *values, const bool *isnull)
void heap_freetuple(HeapTuple htup)
bool index_insert(Relation indexRelation, Datum *values, bool *isnull, ItemPointer heap_t_ctid, Relation heapRelation, IndexUniqueCheck checkUnique, bool indexUnchanged, IndexInfo *indexInfo)
void index_close(Relation relation, LOCKMODE lockmode)
Relation index_open(Oid relationId, LOCKMODE lockmode)
void list_free(List *list)
void pfree(void *pointer)
#define CHECK_FOR_INTERRUPTS()
static int list_length(const List *l)
static Datum PointerGetDatum(const void *X)
static Datum ObjectIdGetDatum(Oid X)
static Pointer DatumGetPointer(Datum X)
static Datum Int32GetDatum(int32 X)
#define RelationGetRelid(relation)
List * RelationGetIndexList(Relation relation)
void ScanKeyInit(ScanKey entry, AttrNumber attributeNumber, StrategyNumber strategy, RegProcedure procedure, Datum argument)
bool HaveRegisteredOrActiveSnapshot(void)
SnapshotData SnapshotToastData
#define BTEqualStrategyNumber
void table_close(Relation relation, LOCKMODE lockmode)
Relation table_open(Oid relationId, LOCKMODE lockmode)
struct varlena * lz4_compress_datum(const struct varlena *value)
struct varlena * pglz_compress_datum(const struct varlena *value)
int default_toast_compression
#define CompressionMethodIsValid(cm)
@ TOAST_INVALID_COMPRESSION_ID
@ TOAST_LZ4_COMPRESSION_ID
@ TOAST_PGLZ_COMPRESSION_ID
#define TOAST_PGLZ_COMPRESSION
#define TOAST_LZ4_COMPRESSION
void toast_close_indexes(Relation *toastidxs, int num_indexes, LOCKMODE lock)
void toast_delete_datum(Relation rel, Datum value, bool is_speculative)
Datum toast_save_datum(Relation rel, Datum value, struct varlena *oldexternal, int options)
Datum toast_compress_datum(Datum value, char cmethod)
Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock)
static bool toastid_valueid_exists(Oid toastrelid, Oid valueid)
static bool toastrel_valueid_exists(Relation toastrel, Oid valueid)
int toast_open_indexes(Relation toastrel, LOCKMODE lock, Relation **toastidxs, int *num_indexes)
Snapshot get_toast_snapshot(void)
#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method)
static bool VARATT_IS_SHORT(const void *PTR)
static Size VARDATA_COMPRESSED_GET_EXTSIZE(const void *PTR)
static bool VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm)
static uint32 VARDATA_COMPRESSED_GET_COMPRESS_METHOD(const void *PTR)
static Size VARSIZE_ANY_EXHDR(const void *PTR)
static bool VARATT_IS_EXTERNAL(const void *PTR)
static char * VARDATA_EXTERNAL(const void *PTR)
static Size VARSIZE(const void *PTR)
static char * VARDATA(const void *PTR)
static void SET_VARTAG_EXTERNAL(void *PTR, vartag_external tag)
static bool VARATT_IS_COMPRESSED(const void *PTR)
static char * VARDATA_SHORT(const void *PTR)
static bool VARATT_EXTERNAL_IS_COMPRESSED(struct varatt_external toast_pointer)
static void SET_VARSIZE(void *PTR, Size len)
static Size VARSIZE_SHORT(const void *PTR)
CommandId GetCurrentCommandId(bool used)