git.postgresql.org Git - postgresql.git/commitdiff

git projects / postgresql.git / commitdiff

Add new GUC, max_worker_processes, limiting number of bgworkers.

author Robert Haas <rhaas@postgresql.org>

Thu, 4 Jul 2013 15:24:24 +0000 (11:24 -0400)

committer Robert Haas <rhaas@postgresql.org>

Thu, 4 Jul 2013 15:24:24 +0000 (11:24 -0400)

In 9.3, there's no particular limit on the number of bgworkers;
instead, we just count up the number that are actually registered,
and use that to set MaxBackends. However, that approach causes
problems for Hot Standby, which needs both MaxBackends and the
size of the lock table to be the same on the standby as on the
master, yet it may not be desirable to run the same bgworkers in
both places. 9.3 handles that by failing to notice the problem,
which will probably work fine in nearly all cases anyway, but is
not theoretically sound.

A further problem with simply counting the number of registered
workers is that new workers can't be registered without a
postmaster restart. This is inconvenient for administrators,
since bouncing the postmaster causes an interruption of service.
Moreover, there are a number of applications for background
processes where, by necessity, the background process must be
started on the fly (e.g. parallel query). While this patch
doesn't actually make it possible to register new background
workers after startup time, it's a necessary prerequisite.

Patch by me. Review by Michael Paquier.

15 files changed:

doc/src/sgml/bgworker.sgml patch | blob | blame | history

doc/src/sgml/config.sgml patch | blob | blame | history

src/backend/access/rmgrdesc/xlogdesc.c patch | blob | blame | history

src/backend/access/transam/xlog.c patch | blob | blame | history

src/backend/postmaster/postmaster.c patch | blob | blame | history

src/backend/storage/lmgr/proc.c patch | blob | blame | history

src/backend/utils/init/globals.c patch | blob | blame | history

src/backend/utils/init/postinit.c patch | blob | blame | history

src/backend/utils/misc/guc.c patch | blob | blame | history

src/backend/utils/misc/postgresql.conf.sample patch | blob | blame | history

src/bin/pg_controldata/pg_controldata.c patch | blob | blame | history

src/bin/pg_resetxlog/pg_resetxlog.c patch | blob | blame | history

src/include/access/xlog_internal.h patch | blob | blame | history

src/include/catalog/pg_control.h patch | blob | blame | history

src/include/miscadmin.h patch | blob | blame | history

diff --git a/doc/src/sgml/bgworker.sgml b/doc/src/sgml/bgworker.sgml

index b0dde7564d86093e9aa4424ab562151e3ebb3f6f..f7126388af18ec4e51582eb4a8ab83e03f87369e 100644 (file)

--- a/doc/src/sgml/bgworker.sgml

+++ b/doc/src/sgml/bgworker.sgml

@@ -146,4 +146,9 @@ typedef struct BackgroundWorker

The <filename>worker_spi</> contrib module contains a working example,

which demonstrates some useful techniques.

</para>

+ <para>

+ The maximum number of registered background workers is limited by

+ <xref linkend="guc-max-worker-processes">.

+ </para>

</chapter>

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml

index 437dbb7711c712a656d01fef6bb883e712f04db9..9126bc37cb54c42a89ab2db0626e5db6fd283d38 100644 (file)

--- a/doc/src/sgml/config.sgml

+++ b/doc/src/sgml/config.sgml

@@ -1595,6 +1595,25 @@ include 'filename'

</para>

</listitem>

</varlistentry>

+ <varlistentry id="guc-max-worker-processes" xreflabel="max_worrker_processes">

+ <term><varname>max_worker_processes</varname> (<type>integer</type>)</term>

+ <indexterm>

+ <primary><varname>max_worker_processes</> configuration parameter</primary>

+ </indexterm>

+ <listitem>

+ <para>

+ Sets the maximum number of background processes that the system

+ can support. This parameter can only be set at server start.

+ </para>

+ <para>

+ When running a standby server, you must set this parameter to the

+ same or higher value than on the master server. Otherwise, queries

+ will not be allowed in the standby server.

+ </para>

+ </listitem>

+ </varlistentry>

</variablelist>

</sect2>

</sect1>

diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c

index 12370521d453beeda4f6e8aa42240bfed26e6e76..1b36f9a88a539b5eac47342f2e11346dde302d2e 100644 (file)

--- a/src/backend/access/rmgrdesc/xlogdesc.c

+++ b/src/backend/access/rmgrdesc/xlogdesc.c

@@ -117,8 +117,9 @@ xlog_desc(StringInfo buf, uint8 xl_info, char *rec)

}

- appendStringInfo(buf, "parameter change: max_connections=%d max_prepared_xacts=%d max_locks_per_xact=%d wal_level=%s",

+ appendStringInfo(buf, "parameter change: max_connections=%d max_worker_processes=%d max_prepared_xacts=%d max_locks_per_xact=%d wal_level=%s",

xlrec.MaxConnections,

+ xlrec.max_worker_processes,

xlrec.max_prepared_xacts,

xlrec.max_locks_per_xact,

wal_level_str);

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c

index 0ce661bf9f4432180b9812b82726f76ebe8745bc..4220859c8a4a26bae0e0f9d4b8eea11cd5ac7672 100644 (file)

--- a/src/backend/access/transam/xlog.c

+++ b/src/backend/access/transam/xlog.c

@@ -4134,6 +4134,7 @@ BootStrapXLOG(void)

/* Set important parameter values for use when replaying WAL */

ControlFile->MaxConnections = MaxConnections;

+ ControlFile->max_worker_processes = max_worker_processes;

ControlFile->max_prepared_xacts = max_prepared_xacts;

ControlFile->max_locks_per_xact = max_locks_per_xact;

ControlFile->wal_level = wal_level;

@@ -4841,6 +4842,9 @@ CheckRequiredParameterValues(void)

RecoveryRequiresIntParameter("max_connections",

MaxConnections,

ControlFile->MaxConnections);

+ RecoveryRequiresIntParameter("max_worker_processes",

+ max_worker_processes,

+ ControlFile->max_worker_processes);

RecoveryRequiresIntParameter("max_prepared_transactions",

max_prepared_xacts,

ControlFile->max_prepared_xacts);

@@ -7770,6 +7774,7 @@ XLogReportParameters(void)

{

if (wal_level != ControlFile->wal_level ||

MaxConnections != ControlFile->MaxConnections ||

+ max_worker_processes != ControlFile->max_worker_processes ||

max_prepared_xacts != ControlFile->max_prepared_xacts ||

max_locks_per_xact != ControlFile->max_locks_per_xact)

{

@@ -7786,6 +7791,7 @@ XLogReportParameters(void)

xl_parameter_change xlrec;

xlrec.MaxConnections = MaxConnections;

+ xlrec.max_worker_processes = max_worker_processes;

xlrec.max_prepared_xacts = max_prepared_xacts;

xlrec.max_locks_per_xact = max_locks_per_xact;

xlrec.wal_level = wal_level;

@@ -7799,6 +7805,7 @@ XLogReportParameters(void)

}

ControlFile->MaxConnections = MaxConnections;

+ ControlFile->max_worker_processes = max_worker_processes;

ControlFile->max_prepared_xacts = max_prepared_xacts;

ControlFile->max_locks_per_xact = max_locks_per_xact;

ControlFile->wal_level = wal_level;

@@ -8184,6 +8191,7 @@ xlog_redo(XLogRecPtr lsn, XLogRecord *record)

LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);

ControlFile->MaxConnections = xlrec.MaxConnections;

+ ControlFile->max_worker_processes = xlrec.max_worker_processes;

ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;

ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;

ControlFile->wal_level = xlrec.wal_level;

diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c

index 15fd4c90eaae1f056b8943bb1471eb1dd8d2aee4..e3b32cbe08044ea3f0797a54e18414013fc8295d 100644 (file)

--- a/src/backend/postmaster/postmaster.c

+++ b/src/backend/postmaster/postmaster.c

@@ -402,7 +402,6 @@ static void reaper(SIGNAL_ARGS);

static void sigusr1_handler(SIGNAL_ARGS);

static void startup_die(SIGNAL_ARGS);

static void dummy_handler(SIGNAL_ARGS);

-static int GetNumRegisteredBackgroundWorkers(int flags);

static void StartupPacketTimeoutHandler(void);

static void CleanupBackend(int pid, int exitstatus);

static bool CleanupBackgroundWorker(int pid, int exitstatus);

@@ -5212,7 +5211,7 @@ int

MaxLivePostmasterChildren(void)

{

return 2 * (MaxConnections + autovacuum_max_workers + 1 +

- GetNumRegisteredBackgroundWorkers(0));

+ max_worker_processes);

}

@@ -5226,7 +5225,6 @@ RegisterBackgroundWorker(BackgroundWorker *worker)

{

RegisteredBgWorker *rw;

int namelen = strlen(worker->bgw_name);

- static int maxworkers;

static int numworkers = 0;

#ifdef EXEC_BACKEND

@@ -5238,11 +5236,6 @@ RegisterBackgroundWorker(BackgroundWorker *worker)

static int BackgroundWorkerCookie = 1;

#endif

- /* initialize upper limit on first call */

- if (numworkers == 0)

- maxworkers = MAX_BACKENDS -

- (MaxConnections + autovacuum_max_workers + 1);

if (!IsUnderPostmaster)

ereport(LOG,

(errmsg("registering background worker: %s", worker->bgw_name)));

@@ -5298,17 +5291,17 @@ RegisterBackgroundWorker(BackgroundWorker *worker)

* Enforce maximum number of workers. Note this is overly restrictive: we

* could allow more non-shmem-connected workers, because these don't count

- * towards the MAX_BACKENDS limit elsewhere. This doesn't really matter

- * for practical purposes; several million processes would need to run on

- * a single server.

+ * towards the MAX_BACKENDS limit elsewhere. For now, it doesn't seem

+ * important to relax this restriction.

- if (++numworkers > maxworkers)

+ if (++numworkers > max_worker_processes)

{

ereport(LOG,

(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),

errmsg("too many background workers"),

errdetail("Up to %d background workers can be registered with the current settings.",

- maxworkers)));

+ max_worker_processes),

+ errhint("Consider increasing the configuration parameter \"max_worker_processes\".")));

return;

}

@@ -5589,41 +5582,6 @@ do_start_bgworker(void)

proc_exit(0);

}

-/*

- * Return the number of background workers registered that have at least

- * one of the passed flag bits set.

- */

-static int

-GetNumRegisteredBackgroundWorkers(int flags)

- slist_iter iter;

- int count = 0;

- slist_foreach(iter, &BackgroundWorkerList)

- {

- RegisteredBgWorker *rw;

- rw = slist_container(RegisteredBgWorker, rw_lnode, iter.cur);

- if (flags != 0 &&

- !(rw->rw_worker.bgw_flags & flags))

- continue;

- count++;

- }

- return count;

-/*

- * Return the number of bgworkers that need to have PGPROC entries.

- */

-int

-GetNumShmemAttachedBgworkers(void)

- return GetNumRegisteredBackgroundWorkers(BGWORKER_SHMEM_ACCESS);

#ifdef EXEC_BACKEND

static pid_t

bgworker_forkexec(int cookie)

diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c

index 6d72a637f740cc4573d305e44fb2b76bd51cd8d6..25bd5285669ded1e2b194b29d2fc327bbda51c7a 100644 (file)

--- a/src/backend/storage/lmgr/proc.c

+++ b/src/backend/storage/lmgr/proc.c

@@ -140,10 +140,8 @@ ProcGlobalSemas(void)

* running out when trying to start another backend is a common failure.

* So, now we grab enough semaphores to support the desired max number

* of backends immediately at initialization --- if the sysadmin has set

- * MaxConnections or autovacuum_max_workers higher than his kernel will

- * support, he'll find out sooner rather than later. (The number of

- * background worker processes registered by loadable modules is also taken

- * into consideration.)

+ * MaxConnections, max_worker_processes, or autovacuum_max_workers higher

+ * than his kernel will support, he'll find out sooner rather than later.

* Another reason for creating semaphores here is that the semaphore

* implementation typically requires us to create semaphores in the

diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c

index 9f51929191d5b23d12e13068843117a988df3436..33efb3c3cca933f69aa16286aa8c0e68d67bb754 100644 (file)

--- a/src/backend/utils/init/globals.c

+++ b/src/backend/utils/init/globals.c

@@ -109,6 +109,7 @@ int maintenance_work_mem = 16384;

int NBuffers = 1000;

int MaxConnections = 90;

+int max_worker_processes = 8;

int MaxBackends = 0;

int VacuumCostPageHit = 1; /* GUC parameters for vacuum */

diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c

index 127f9273e85795f37ce6bbe7807b926296e38886..2c7f0f1764113602504386f4972371a97e8fa332 100644 (file)

--- a/src/backend/utils/init/postinit.c

+++ b/src/backend/utils/init/postinit.c

@@ -436,7 +436,7 @@ InitializeMaxBackends(void)

/* the extra unit accounts for the autovacuum launcher */

MaxBackends = MaxConnections + autovacuum_max_workers + 1 +

- GetNumShmemAttachedBgworkers();

+ + max_worker_processes;

/* internal error because the values were all checked previously */

if (MaxBackends > MAX_BACKENDS)

diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c

index 3a7653698d35864d2f4851e55acdb84ba3d271d3..d6200616de8df517096fa9f9540887ad1a476439 100644 (file)

--- a/src/backend/utils/misc/guc.c

+++ b/src/backend/utils/misc/guc.c

@@ -190,6 +190,7 @@ static const char *show_tcp_keepalives_idle(void);

static const char *show_tcp_keepalives_interval(void);

static const char *show_tcp_keepalives_count(void);

static bool check_maxconnections(int *newval, void **extra, GucSource source);

+static bool check_max_worker_processes(int *newval, void **extra, GucSource source);

static bool check_autovacuum_max_workers(int *newval, void **extra, GucSource source);

static bool check_effective_io_concurrency(int *newval, void **extra, GucSource source);

static void assign_effective_io_concurrency(int newval, void *extra);

@@ -2158,6 +2159,18 @@ static struct config_int ConfigureNamesInt[] =

check_effective_io_concurrency, assign_effective_io_concurrency, NULL

+ {

+ {"max_worker_processes",

+ PGC_POSTMASTER,

+ RESOURCES_ASYNCHRONOUS,

+ gettext_noop("Maximum number of concurrent worker processes."),

+ NULL,

+ },

+ &max_worker_processes,

+ 8, 1, MAX_BACKENDS,

+ check_max_worker_processes, NULL, NULL

+ },

{

{"log_rotation_age", PGC_SIGHUP, LOGGING_WHERE,

gettext_noop("Automatic log file rotation will occur after N minutes."),

@@ -8667,8 +8680,8 @@ show_tcp_keepalives_count(void)

static bool

check_maxconnections(int *newval, void **extra, GucSource source)

{

- if (*newval + GetNumShmemAttachedBgworkers() + autovacuum_max_workers + 1 >

- MAX_BACKENDS)

+ if (*newval + autovacuum_max_workers + 1 +

+ max_worker_processes > MAX_BACKENDS)

return false;

return true;

}

@@ -8676,8 +8689,15 @@ check_maxconnections(int *newval, void **extra, GucSource source)

static bool

check_autovacuum_max_workers(int *newval, void **extra, GucSource source)

{

- if (MaxConnections + *newval + 1 + GetNumShmemAttachedBgworkers() >

- MAX_BACKENDS)

+ if (MaxConnections + *newval + 1 + max_worker_processes > MAX_BACKENDS)

+ return false;

+ return true;

+static bool

+check_max_worker_processes(int *newval, void **extra, GucSource source)

+ if (MaxConnections + autovacuum_max_workers + 1 + *newval > MAX_BACKENDS)

return false;

return true;

}

diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample

index 0d7249f4dbd7421d1184cbfaffb9d384071da5fe..d69a02be87f7ffdc8e0715c379ba7e0c5fa78376 100644 (file)

--- a/src/backend/utils/misc/postgresql.conf.sample

+++ b/src/backend/utils/misc/postgresql.conf.sample

@@ -152,6 +152,7 @@

# - Asynchronous Behavior -

#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching

+#max_worker_processes = 8

#------------------------------------------------------------------------------

diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c

index a790f99cb51d3649d8bbd2f32ab0065470bccda8..fde483a616ab4092d05eb0a1c0ac7b981c4b2458 100644 (file)

--- a/src/bin/pg_controldata/pg_controldata.c

+++ b/src/bin/pg_controldata/pg_controldata.c

@@ -260,6 +260,8 @@ main(int argc, char *argv[])

wal_level_str(ControlFile.wal_level));

printf(_("Current max_connections setting: %d\n"),

ControlFile.MaxConnections);

+ printf(_("Current max_worker_processes setting: %d\n"),

+ ControlFile.max_worker_processes);

printf(_("Current max_prepared_xacts setting: %d\n"),

ControlFile.max_prepared_xacts);

printf(_("Current max_locks_per_xact setting: %d\n"),

diff --git a/src/bin/pg_resetxlog/pg_resetxlog.c b/src/bin/pg_resetxlog/pg_resetxlog.c

index 6d3e9f5439f652f1ddac73c0c5b4d86bc7d571c7..465905c8503126037930450a4b34bc31020752be 100644 (file)

--- a/src/bin/pg_resetxlog/pg_resetxlog.c

+++ b/src/bin/pg_resetxlog/pg_resetxlog.c

@@ -518,6 +518,7 @@ GuessControlValues(void)

ControlFile.wal_level = WAL_LEVEL_MINIMAL;

ControlFile.MaxConnections = 100;

+ ControlFile.max_worker_processes = 8;

ControlFile.max_prepared_xacts = 0;

ControlFile.max_locks_per_xact = 64;

@@ -664,6 +665,7 @@ RewriteControlFile(void)

ControlFile.wal_level = WAL_LEVEL_MINIMAL;

ControlFile.MaxConnections = 100;

+ ControlFile.max_worker_processes = 8;

ControlFile.max_prepared_xacts = 0;

ControlFile.max_locks_per_xact = 64;

diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h

index ee12d1a110a2a55db98e7efaa6a232238deb4ade..c3e173106fc032479d103f94c6bf76df5b3be109 100644 (file)

--- a/src/include/access/xlog_internal.h

+++ b/src/include/access/xlog_internal.h

@@ -55,7 +55,7 @@ typedef struct BkpBlock

* Each page of XLOG file has a header like this:

-#define XLOG_PAGE_MAGIC 0xD075 /* can be used as WAL version indicator */

+#define XLOG_PAGE_MAGIC 0xD076 /* can be used as WAL version indicator */

typedef struct XLogPageHeaderData

{

@@ -205,6 +205,7 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;

typedef struct xl_parameter_change

{

int MaxConnections;

+ int max_worker_processes;

int max_prepared_xacts;

int max_locks_per_xact;

int wal_level;

diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h

index 0e297610d8a526847b260ea31db28f54056a1023..637221e6347c1e673a24675176dce751caa9d872 100644 (file)

--- a/src/include/catalog/pg_control.h

+++ b/src/include/catalog/pg_control.h

@@ -172,6 +172,7 @@ typedef struct ControlFileData

int wal_level;

int MaxConnections;

+ int max_worker_processes;

int max_prepared_xacts;

int max_locks_per_xact;

diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h

index be3add95dcaa4bc5bd98451651d1b6bea125139d..48985b370fa6fe8916c16ec66b346765cdc92b0c 100644 (file)

--- a/src/include/miscadmin.h

+++ b/src/include/miscadmin.h

@@ -141,6 +141,7 @@ extern PGDLLIMPORT char *DataDir;

extern PGDLLIMPORT int NBuffers;

extern int MaxBackends;

extern int MaxConnections;

+extern int max_worker_processes;

extern PGDLLIMPORT int MyProcPid;

extern PGDLLIMPORT pg_time_t MyStartTime;

This is the main PostgreSQL git repository.

RSS Atom