journald: log when we fail to forward messages to syslog

https://bugzilla.redhat.com/show_bug.cgi?id=847207

TODO

@@ -24,18 +24,16 @@ Bugfixes:
 
 F18:
 
-* journald: warn if we drop messages we forward to the syslog socket
-
 * refuse automount triggers when automount is queued for stop, much like we refuse socket triggers when sockets are queued for stop
 
-* There's something wrong with escaping unit names: http://lists.freedesktop.org/archives/systemd-devel/2012-August/006292.html
-
 * logind: different policy actions for idle, suspend, shutdown blockers: allow idle blockers by default, don't allow suspend blockers by default
 
 * selinux: merge systemd selinux access controls (dwalsh)
 
 Features:
 
+* instantiated target units
+
 * support *static* (/run) hibernate inhibitors. All rpm -i actions should completely prevent any
   sort of hibernate action until the next reboot. If the kernel or any other base tool is replaced
   by rpm, the resume path might fail, the for resume needed kernel might even be uninstalled, and

src/journal/journald-syslog.c

@@ -23,12 +23,16 @@
 #include <stddef.h>
 #include <sys/epoll.h>
 
+#include "systemd/sd-messages.h"
 #include "socket-util.h"
 #include "journald.h"
 #include "journald-syslog.h"
 #include "journald-kmsg.h"
 #include "journald-console.h"
 
+/* Warn once every 30s if we missed syslog message */
+#define WARN_FORWARD_SYSLOG_MISSED_USEC (30 * USEC_PER_SEC)
+
 static void forward_syslog_iovec(Server *s, const struct iovec *iovec, unsigned n_iovec, struct ucred *ucred, struct timeval *tv) {
         struct msghdr msghdr;
         struct cmsghdr *cmsg;
@@ -74,8 +78,10 @@ static void forward_syslog_iovec(Server *s, const struct iovec *iovec, unsigned
 
         /* The socket is full? I guess the syslog implementation is
          * too slow, and we shouldn't wait for that... */
-        if (errno == EAGAIN)
+        if (errno == EAGAIN) {
+                s->n_forward_syslog_missed++;
                 return;
+        }
 
         if (ucred && errno == ESRCH) {
                 struct ucred u;
@@ -91,8 +97,10 @@ static void forward_syslog_iovec(Server *s, const struct iovec *iovec, unsigned
                 if (sendmsg(s->syslog_fd, &msghdr, MSG_NOSIGNAL) >= 0)
                         return;
 
-                if (errno == EAGAIN)
+                if (errno == EAGAIN) {
+                        s->n_forward_syslog_missed++;
                         return;
+                }
         }
 
         if (errno != ENOENT)
@@ -464,3 +472,20 @@ int server_open_syslog_socket(Server *s) {
 
         return 0;
 }
+
+void server_maybe_warn_forward_syslog_missed(Server *s) {
+        usec_t n;
+        assert(s);
+
+        if (s->n_forward_syslog_missed <= 0)
+                return;
+
+        n = now(CLOCK_MONOTONIC);
+        if (s->last_warn_forward_syslog_missed + WARN_FORWARD_SYSLOG_MISSED_USEC > n)
+                return;
+
+        server_driver_message(s, SD_MESSAGE_FORWARD_SYSLOG_MISSED, "Forwarding to syslog missed %u messages.", s->n_forward_syslog_missed);
+
+        s->n_forward_syslog_missed = 0;
+        s->last_warn_forward_syslog_missed = n;
+}

src/journal/journald-syslog.h

@@ -32,3 +32,5 @@ void server_forward_syslog(Server *s, int priority, const char *identifier, cons
 
 void server_process_syslog_message(Server *s, const char *buf, struct ucred *ucred, struct timeval *tv, const char *label, size_t label_len);
 int server_open_syslog_socket(Server *s);
+
+void server_maybe_warn_forward_syslog_missed(Server *s);

src/journal/journald.c

@@ -1553,6 +1553,7 @@ int main(int argc, char *argv[]) {
                 }
 
                 server_maybe_append_tags(&server);
+                server_maybe_warn_forward_syslog_missed(&server);
         }
 
         log_debug("systemd-journald stopped as pid %lu", (unsigned long) getpid());

src/journal/journald.h

@@ -82,6 +82,9 @@ typedef struct Server {
         bool forward_to_syslog;
         bool forward_to_console;
 
+        unsigned n_forward_syslog_missed;
+        usec_t last_warn_forward_syslog_missed;
+
         uint64_t cached_available_space;
         usec_t cached_available_space_timestamp;
 

src/systemd/sd-messages.h

@@ -58,6 +58,8 @@ extern "C" {
 #define SD_MESSAGE_UNIT_RELOADING   SD_ID128_MAKE(d3,4d,03,7f,ff,18,47,e6,ae,66,9a,37,0e,69,47,25)
 #define SD_MESSAGE_UNIT_RELOADED    SD_ID128_MAKE(7b,05,eb,c6,68,38,42,22,ba,a8,88,11,79,cf,da,54)
 
+#define SD_MESSAGE_FORWARD_SYSLOG_MISSED SD_ID128_MAKE(00,27,22,9c,a0,64,41,81,a7,6c,4e,92,45,8a,fa,2e)
+
 #ifdef __cplusplus
 }
 #endif