diff -rpcd orig/doc/src/sgml/runtime.sgml new/doc/src/sgml/runtime.sgml *** orig/doc/src/sgml/runtime.sgml 2013-05-07 05:57:06.000000000 +0900 --- new/doc/src/sgml/runtime.sgml 2013-06-27 10:49:09.000000000 +0900 *************** echo -1000 > /proc/self/oom_score_adj *** 1362,1372 **** This is the Immediate Shutdown mode. ! The master postgres process will send a ! SIGQUIT to all child processes and exit ! immediately, without properly shutting itself down. The child processes ! likewise exit immediately upon receiving ! SIGQUIT. This will lead to recovery (by replaying the WAL log) upon next start-up. This is recommended only in emergencies. --- 1362,1372 ---- This is the Immediate Shutdown mode. ! The server will send SIGQUIT to all child ! processes and wait for them to terminate. Those that don't terminate ! within 5 seconds, will be sent SIGKILL by the ! master postgres process, which will then terminate ! without further waiting. This will lead to recovery (by replaying the WAL log) upon next start-up. This is recommended only in emergencies. diff -rpcd orig/src/backend/postmaster/postmaster.c new/src/backend/postmaster/postmaster.c *** orig/src/backend/postmaster/postmaster.c 2013-05-07 05:57:06.000000000 +0900 --- new/src/backend/postmaster/postmaster.c 2013-06-27 10:49:25.000000000 +0900 *************** static pid_t StartupPID = 0, *** 273,278 **** --- 273,279 ---- #define NoShutdown 0 #define SmartShutdown 1 #define FastShutdown 2 + #define ImmediateShutdown 3 static int Shutdown = NoShutdown; *************** typedef enum *** 343,348 **** --- 344,353 ---- static PMState pmState = PM_INIT; + /* Start time of abort processing at immediate shutdown or child crash */ + static time_t AbortStartTime; + #define SIGKILL_CHILDREN_AFTER_SECS 5 + static bool ReachedNormalRunning = false; /* T if we've reached PM_RUN */ bool ClientAuthInProgress = false; /* T during new-client *************** static void RandomSalt(char *md5Salt); *** 419,424 **** --- 424,430 ---- static void signal_child(pid_t pid, int signal); static bool SignalSomeChildren(int signal, int targets); static bool SignalUnconnectedWorkers(int signal); + static void TerminateChildren(int signal); #define SignalChildren(sig) SignalSomeChildren(sig, BACKEND_TYPE_ALL) *************** DetermineSleepTime(struct timeval *timeo *** 1425,1432 **** if (Shutdown > NoShutdown || (!StartWorkerNeeded && !HaveCrashedWorker)) { ! timeout->tv_sec = 60; ! timeout->tv_usec = 0; return; } --- 1431,1446 ---- if (Shutdown > NoShutdown || (!StartWorkerNeeded && !HaveCrashedWorker)) { ! if (AbortStartTime > 0) ! { ! timeout->tv_sec = SIGKILL_CHILDREN_AFTER_SECS; ! timeout->tv_usec = 0; ! } ! else ! { ! timeout->tv_sec = 60; ! timeout->tv_usec = 0; ! } return; } *************** ServerLoop(void) *** 1658,1663 **** --- 1672,1699 ---- TouchSocketLockFiles(); last_touch_time = now; } + + /* + * If we already sent SIGQUIT to children and they are slow to shut + * down, it's time to send them SIGKILL. This doesn't happen normally, + * but under certain conditions backends can get stuck while shutting + * down. This is a last measure to get them unwedged. + * + * Note we also do this during recovery from a process crash. + */ + if ((Shutdown >= ImmediateShutdown || (FatalError && !SendStop)) && + now - AbortStartTime >= SIGKILL_CHILDREN_AFTER_SECS) + { + /* We were gentle with them before. Not anymore */ + TerminateChildren(SIGKILL); + + /* + * Additionally, unless we're recovering from a process crash, it's + * now the time for postmaster to abandon ship. + */ + if (!FatalError) + ExitPostmaster(1); + } } } *************** pmdie(SIGNAL_ARGS) *** 2453,2482 **** /* * Immediate Shutdown: * ! * abort all children with SIGQUIT and exit without attempt to ! * properly shut down data base system. */ ereport(LOG, (errmsg("received immediate shutdown request"))); ! SignalChildren(SIGQUIT); ! if (StartupPID != 0) ! signal_child(StartupPID, SIGQUIT); ! if (BgWriterPID != 0) ! signal_child(BgWriterPID, SIGQUIT); ! if (CheckpointerPID != 0) ! signal_child(CheckpointerPID, SIGQUIT); ! if (WalWriterPID != 0) ! signal_child(WalWriterPID, SIGQUIT); ! if (WalReceiverPID != 0) ! signal_child(WalReceiverPID, SIGQUIT); ! if (AutoVacPID != 0) ! signal_child(AutoVacPID, SIGQUIT); ! if (PgArchPID != 0) ! signal_child(PgArchPID, SIGQUIT); ! if (PgStatPID != 0) ! signal_child(PgStatPID, SIGQUIT); ! SignalUnconnectedWorkers(SIGQUIT); ! ExitPostmaster(0); break; } --- 2489,2515 ---- /* * Immediate Shutdown: * ! * abort all children with SIGQUIT, wait for them to exit, ! * terminate remaining ones with SIGKILL, then exit without ! * attempt to properly shut down the data base system. */ + if (Shutdown >= ImmediateShutdown) + break; + Shutdown = ImmediateShutdown; ereport(LOG, (errmsg("received immediate shutdown request"))); ! ! TerminateChildren(SIGQUIT); ! pmState = PM_WAIT_BACKENDS; ! ! /* set stopwatch for them to die */ ! AbortStartTime = time(NULL); ! ! /* ! * Now wait for backends to exit. If there are none, ! * PostmasterStateMachine will take the next step. ! */ ! PostmasterStateMachine(); break; } *************** HandleChildCrash(int pid, int exitstatus *** 2955,2961 **** * Make log entry unless there was a previous crash (if so, nonzero exit * status is to be expected in SIGQUIT response; don't clutter log) */ ! if (!FatalError) { LogChildExit(LOG, procname, pid, exitstatus); ereport(LOG, --- 2988,2994 ---- * Make log entry unless there was a previous crash (if so, nonzero exit * status is to be expected in SIGQUIT response; don't clutter log) */ ! if (!FatalError && Shutdown != ImmediateShutdown) { LogChildExit(LOG, procname, pid, exitstatus); ereport(LOG, *************** HandleChildCrash(int pid, int exitstatus *** 3001,3007 **** * (-s on command line), then we send SIGSTOP instead, so that we * can get core dumps from all backends by hand. */ ! if (!FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3034,3040 ---- * (-s on command line), then we send SIGSTOP instead, so that we * can get core dumps from all backends by hand. */ ! if (!FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3053,3059 **** if (bp->bkend_type == BACKEND_TYPE_BGWORKER) continue; ! if (!FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3086,3092 ---- if (bp->bkend_type == BACKEND_TYPE_BGWORKER) continue; ! if (!FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3067,3073 **** /* Take care of the startup process too */ if (pid == StartupPID) StartupPID = 0; ! else if (StartupPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3100,3106 ---- /* Take care of the startup process too */ if (pid == StartupPID) StartupPID = 0; ! else if (StartupPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3079,3085 **** /* Take care of the bgwriter too */ if (pid == BgWriterPID) BgWriterPID = 0; ! else if (BgWriterPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3112,3118 ---- /* Take care of the bgwriter too */ if (pid == BgWriterPID) BgWriterPID = 0; ! else if (BgWriterPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3091,3097 **** /* Take care of the checkpointer too */ if (pid == CheckpointerPID) CheckpointerPID = 0; ! else if (CheckpointerPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3124,3130 ---- /* Take care of the checkpointer too */ if (pid == CheckpointerPID) CheckpointerPID = 0; ! else if (CheckpointerPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3103,3109 **** /* Take care of the walwriter too */ if (pid == WalWriterPID) WalWriterPID = 0; ! else if (WalWriterPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3136,3142 ---- /* Take care of the walwriter too */ if (pid == WalWriterPID) WalWriterPID = 0; ! else if (WalWriterPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3115,3121 **** /* Take care of the walreceiver too */ if (pid == WalReceiverPID) WalReceiverPID = 0; ! else if (WalReceiverPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3148,3154 ---- /* Take care of the walreceiver too */ if (pid == WalReceiverPID) WalReceiverPID = 0; ! else if (WalReceiverPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3127,3133 **** /* Take care of the autovacuum launcher too */ if (pid == AutoVacPID) AutoVacPID = 0; ! else if (AutoVacPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3160,3166 ---- /* Take care of the autovacuum launcher too */ if (pid == AutoVacPID) AutoVacPID = 0; ! else if (AutoVacPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3142,3148 **** * simplifies the state-machine logic in the case where a shutdown request * arrives during crash processing.) */ ! if (PgArchPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3175,3181 ---- * simplifies the state-machine logic in the case where a shutdown request * arrives during crash processing.) */ ! if (PgArchPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3157,3163 **** * simplifies the state-machine logic in the case where a shutdown request * arrives during crash processing.) */ ! if (PgStatPID != 0 && !FatalError) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", --- 3190,3196 ---- * simplifies the state-machine logic in the case where a shutdown request * arrives during crash processing.) */ ! if (PgStatPID != 0 && !FatalError && Shutdown != ImmediateShutdown) { ereport(DEBUG2, (errmsg_internal("sending %s to process %d", *************** HandleChildCrash(int pid, int exitstatus *** 3169,3175 **** /* We do NOT restart the syslogger */ ! FatalError = true; /* We now transit into a state of waiting for children to die */ if (pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || --- 3202,3209 ---- /* We do NOT restart the syslogger */ ! if (Shutdown != ImmediateShutdown) ! FatalError = true; /* We now transit into a state of waiting for children to die */ if (pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || *************** HandleChildCrash(int pid, int exitstatus *** 3178,3183 **** --- 3212,3224 ---- pmState == PM_WAIT_READONLY || pmState == PM_SHUTDOWN) pmState = PM_WAIT_BACKENDS; + + /* + * .. and if this doesn't happen quickly enough, now the clock is ticking + * for us to kill them without mercy. + */ + if (AbortStartTime == 0) + AbortStartTime = time(NULL); } /* *************** PostmasterStateMachine(void) *** 3314,3320 **** WalWriterPID == 0 && AutoVacPID == 0) { ! if (FatalError) { /* * Start waiting for dead_end children to die. This state --- 3355,3361 ---- WalWriterPID == 0 && AutoVacPID == 0) { ! if (Shutdown >= ImmediateShutdown || FatalError) { /* * Start waiting for dead_end children to die. This state *************** PostmasterStateMachine(void) *** 3324,3330 **** /* * We already SIGQUIT'd the archiver and stats processes, if ! * any, when we entered FatalError state. */ } else --- 3365,3372 ---- /* * We already SIGQUIT'd the archiver and stats processes, if ! * any, when we started immediate shutdown or entered ! * FatalError state. */ } else *************** signal_child(pid_t pid, int signal) *** 3509,3514 **** --- 3551,3557 ---- case SIGTERM: case SIGQUIT: case SIGSTOP: + case SIGKILL: if (kill(-pid, signal) < 0) elog(DEBUG3, "kill(%ld,%d) failed: %m", (long) (-pid), signal); break; *************** SignalSomeChildren(int signal, int targe *** 3596,3601 **** --- 3639,3671 ---- } /* + * Send a termination signal to children. This considers all of our children + * processes, except syslogger and dead_end backends. + */ + static void + TerminateChildren(int signal) + { + SignalChildren(signal); + if (StartupPID != 0) + signal_child(StartupPID, signal); + if (BgWriterPID != 0) + signal_child(BgWriterPID, signal); + if (CheckpointerPID != 0) + signal_child(CheckpointerPID, signal); + if (WalWriterPID != 0) + signal_child(WalWriterPID, signal); + if (WalReceiverPID != 0) + signal_child(WalReceiverPID, signal); + if (AutoVacPID != 0) + signal_child(AutoVacPID, signal); + if (PgArchPID != 0) + signal_child(PgArchPID, signal); + if (PgStatPID != 0) + signal_child(PgStatPID, signal); + SignalUnconnectedWorkers(signal); + } + + /* * BackendStartup -- start backend process * * returns: STATUS_ERROR if the fork failed, STATUS_OK otherwise. diff -rpcd orig/src/port/kill.c new/src/port/kill.c *** orig/src/port/kill.c 2013-05-07 05:57:06.000000000 +0900 --- new/src/port/kill.c 2013-06-27 10:49:09.000000000 +0900 *************** pgkill(int pid, int sig) *** 38,43 **** --- 38,63 ---- errno = EINVAL; return -1; } + + /* special case for SIGKILL: just ask the system to terminate the target */ + if (sig == SIGKILL) + { + HANDLE prochandle; + + if ((prochandle = OpenProcess(PROCESS_TERMINATE, FALSE, (DWORD) pid)) == NULL) + { + errno = ESRCH; + return -1; + } + if (!TerminateProcess(prochandle, 255)) + { + _dosmaperr(GetLastError()); + CloseHandle(prochandle); + return -1; + } + CloseHandle(prochandle); + return 0; + } snprintf(pipename, sizeof(pipename), "\\\\.\\pipe\\pgsignal_%u", pid); if (CallNamedPipe(pipename, &sigData, 1, &sigRet, 1, &bytes, 1000))