1/*-------------------------------------------------------------------------
4 * Routines for archivers to write an uncompressed or compressed data
7 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
8 * Portions Copyright (c) 1994, Regents of the University of California
10 * This file includes two APIs for dealing with compressed data. The first
11 * provides more flexibility, using callbacks to read/write data from the
12 * underlying stream. The second API is a wrapper around fopen and
13 * friends, providing an interface similar to those, but abstracts away
14 * the possible compression. The second API is aimed for the resulting
15 * files to be easily manipulated with an external compression utility
21 * The interface for writing to an archive consists of three functions:
22 * AllocateCompressor, writeData, and EndCompressor. First you call
23 * AllocateCompressor, then write all the data by calling writeData as many
24 * times as needed, and finally EndCompressor. writeData will call the
25 * WriteFunc that was provided to AllocateCompressor for each chunk of
28 * The interface for reading an archive consists of the same three functions:
29 * AllocateCompressor, readData, and EndCompressor. First you call
30 * AllocateCompressor, then read all the data by calling readData to read the
31 * whole compressed stream which repeatedly calls the given ReadFunc. ReadFunc
32 * returns the compressed data one chunk at a time. Then readData decompresses
33 * it and passes the decompressed data to ahwrite(), until ReadFunc returns 0
34 * to signal EOF. The interface is the same for compressed and uncompressed
37 * Compressed stream API
38 * ----------------------
40 * The compressed stream API is providing a set of function pointers for
41 * opening, reading, writing, and finally closing files. The implemented
42 * function pointers are documented in the corresponding header file and are
43 * common for all streams. It allows the caller to use the same functions for
44 * both compressed and uncompressed streams.
46 * The interface consists of three functions, InitCompressFileHandle,
47 * InitDiscoverCompressFileHandle, and EndCompressFileHandle. If the
48 * compression is known, then start by calling InitCompressFileHandle,
49 * otherwise discover it by using InitDiscoverCompressFileHandle. Then call
50 * the function pointers as required for the read/write operations. Finally
51 * call EndCompressFileHandle to end the stream.
53 * InitDiscoverCompressFileHandle tries to infer the compression by the
54 * filename suffix. If the suffix is not yet known then it tries to simply
55 * open the file and if it fails, it tries to open the same file with
56 * compressed suffixes (.gz, .lz4 and .zst, in this order).
59 * src/bin/pg_dump/compress_io.c
61 *-------------------------------------------------------------------------
74/*----------------------
76 *----------------------
80 * Checks whether support for a compression algorithm is implemented in
83 * On success returns NULL, otherwise returns a malloc'ed string which can be
84 * used by the caller in an error message.
90 bool supported =
false;
108 return psprintf(
_(
"this build does not support compression with %s"),
114/*----------------------
116 *----------------------
120 * Allocate a new compressor.
145 * Terminate compression library context and flush its buffers.
154/*----------------------
155 * Compressed stream API
156 *----------------------
166 int suffixlen = strlen(suffix);
168 if (filenamelen < suffixlen)
171 return memcmp(&
filename[filenamelen - suffixlen],
176/* free() without changing errno; useful in several places below */
180 int save_errno = errno;
191 * Initialize a compress file handle for the specified compression algorithm.
213 * Checks if a compressed file (with the specified extension) exists.
215 * The filename of the tested file is stored to fname buffer (the existing
216 * buffer is freed, new buffer is allocated and returned through the pointer).
222 *fname =
psprintf(
"%s.%s", path, ext);
223 return (
access(*fname, F_OK) == 0);
227 * Open a file for reading. 'path' is the file to open, and 'mode' should
228 * be either "r" or "rb".
230 * If the file at 'path' contains the suffix of a supported compression method,
231 * currently this includes ".gz", ".lz4" and ".zst", then this compression will be used
232 * throughout. Otherwise the compression will be inferred by iteratively trying
233 * to open the file at 'path', first as is, then by appending known compression
234 * suffixes. So if you pass "foo" as 'path', this will open either "foo" or
235 * "foo.{gz,lz4,zst}", trying in that order.
237 * On failure, return NULL with an error code in errno.
261 if (
stat(path, &st) == 0)
284 * Close an open file handle and release its memory.
286 * On failure, returns false and sets errno appropriately.
void InitCompressFileHandleGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
void InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
static void free_keep_errno(void *p)
static int hasSuffix(const char *filename, const char *suffix)
bool EndCompressFileHandle(CompressFileHandle *CFH)
CompressorState * AllocateCompressor(const pg_compress_specification compression_spec, ReadFunc readF, WriteFunc writeF)
CompressFileHandle * InitDiscoverCompressFileHandle(const char *path, const char *mode)
char * supports_compression(const pg_compress_specification compression_spec)
void EndCompressor(ArchiveHandle *AH, CompressorState *cs)
CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compression_spec)
static bool check_compressed_file(const char *path, char **fname, char *ext)
size_t(* ReadFunc)(ArchiveHandle *AH, char **buf, size_t *buflen)
void(* WriteFunc)(ArchiveHandle *AH, const char *buf, size_t len)
void InitCompressFileHandleLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
void InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec)
void InitCompressorNone(CompressorState *cs, const pg_compress_specification compression_spec)
void InitCompressFileHandleNone(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
void InitCompressorZstd(CompressorState *cs, const pg_compress_specification compression_spec)
void InitCompressFileHandleZstd(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
const char * get_compress_algorithm_name(pg_compress_algorithm algorithm)
char * pg_strdup(const char *in)
void * pg_malloc0(size_t size)
Assert(PointerIsAligned(start, uint64))
static PgChecksumMode mode
char * psprintf(const char *fmt,...)
bool(* open_func)(const char *path, int fd, const char *mode, CompressFileHandle *CFH)
bool(* close_func)(CompressFileHandle *CFH)
void(* end)(ArchiveHandle *AH, CompressorState *cs)
pg_compress_algorithm algorithm