From: Matthew Grosso < mgrosso at looksmart dot net>
To: cronolog-users@icarus.demon.co.uk
Subject: CRONOLOG: jumbo patch, includes launching external program to handle logs
Date: Fri, 10 Oct 2003 13:02:14 -0400

I've put together a jumbo patch with some features that are on the todo 
list and most of the outstanding patches on the site. specifically:

the todo list items completed (which motivated the work for my companies use) are:

- ability to launch a helper program on every log file after it is closed 
    if multiple cronologs writing to the same file, only one helper is  
    launched.

- signal handling to ensure that every log file has a chance to be handled
    by that helper and that cronolog doesnt ever hang apache restarts

- alarm() used so log files rotate on time period boundary even if there is no
  traffic

I also implemented (with tweaks) many of the outstanding patches, some of
which are also on the todo list:

- uid/gid patch from Isaac Wilcox <iwilcox at eatstatic dot net>
    minus the configure changes which broke the build for me.

- the large file patch from matt lanier <mlanier at danger dot .com>
    with tweaks from me to work with my own changes above.

- incorporated a slightly different implementation of
    the SIGUSR1 patch from Prakash Kailasa <PKailasa at seisint dot com>

- the s/stat/lstat/ fix from Victor Martinez <victor at kablinkteam dot com>

the changes have only been tested on my Linux 2.4 box, and should be considered
beta until more users have tried it. That said, we'll be starting to qa this at
looksmart soon, but again, only on linux.

I'm including a sample script that can be used as a helper. It will first
compress, then scp a log to multiple destinations, then remove it.

Still todo: I need to put my new options into the man page, not just the usage,
and see if the configure.in needs modification.

the patch is available below, and at this url:
http://www.falconweb.com/~mattg/code/cronolog-1.6.2-jumbo-20031010.diff

diff -Nur cronolog-1.6.2/src/cronolog.c cronolog-1.6.2-jumbo-20031008/src/cronolog.c
--- cronolog-1.6.2/src/cronolog.c	2001-05-03 12:42:48.000000000 -0400
+++ cronolog-1.6.2-jumbo-20031008/src/cronolog.c	2003-10-08 00:22:10.000000000 -0400
@@ -82,6 +82,18 @@
  * written to "file" (e.g. /dev/console) or to stderr if "file" is "-".
  */
 
+#ifndef _WIN32
+#define _GNU_SOURCE 1
+#define OPEN_EXCLUSIVE O_WRONLY|O_CREAT|O_EXCL|O_APPEND|O_LARGEFILE
+#define OPEN_SHARED O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE
+#else
+#define OPEN_EXCLUSIVE O_WRONLY|O_CREAT|O_EXCL|O_APPEND
+#define OPEN_SHARED O_WRONLY|O_CREAT|O_APPEND
+#endif
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <signal.h>
 #include "cronoutils.h"
 #include "getopt.h"
 
@@ -91,6 +103,16 @@
 int	new_log_file(const char *, const char *, mode_t, const char *,
 		     PERIODICITY, int, int, char *, size_t, time_t, time_t *);
 
+void	cleanup(int );
+void	handle_file();
+void	fork_to_handle_file();
+
+int     openwrapper( const char *filename );
+
+#ifndef _WIN32
+void    setsig_handler( int signum,  void (*action)(int, siginfo_t *, void *));
+void    set_signal_handlers();
+#endif
 
 /* Definition of version and usage messages */
 
@@ -100,6 +122,12 @@
 #define VERSION_MSG      "cronolog version 0.1\n"
 #endif
 
+#ifndef _WIN32
+#define SETUGID_USAGE	"   -u USER,   --set-uid=USER  change to USER before doing anything (name or UID)\n" \
+			"   -g GROUP,  --set-gid=GROUP change to GROUP before doing anything (name or GID)\n"
+#else
+#define SETUGID_USAGE	""
+#endif
 
 #define USAGE_MSG 	"usage: %s [OPTIONS] logfile-spec\n" \
 			"\n" \
@@ -113,6 +141,11 @@
 			"   -o,        --once-only     create single output log from template (not rotated)\n" \
 			"   -x FILE,   --debug=FILE    write debug messages to FILE\n" \
 			"                              ( or to standard error if FILE is \"-\")\n" \
+			"   -r,        --helper=SCRIPT post rotation helper script to fork exec on old files\n" \
+			"                              ( will be called like \"SCRIPT <oldlog>\" )\n" \
+			"                              ( not tested on windows )\n" \
+			"   -G,        --helper-arg=ARG argument passed to rotation helper script\n" \
+            SETUGID_USAGE \
 			"   -a,        --american         American date formats\n" \
 			"   -e,        --european         European date formats (default)\n" \
 			"   -s,    --start-time=TIME   starting time\n" \
@@ -122,7 +155,7 @@
 
 /* Definition of the short and long program options */
 
-char          *short_options = "ad:eop:s:z:H:P:S:l:hVx:";
+char          *short_options = "ad:eop:s:z:H:P:S:l:hVx:r:G:u:g:";
 
 #ifndef _WIN32
 struct option long_options[] =
@@ -137,12 +170,23 @@
     { "link",      	required_argument, 	NULL, 'l' },
     { "period",		required_argument,	NULL, 'p' },
     { "delay",		required_argument,	NULL, 'd' },
+    { "helper",		required_argument,	NULL, 'r' },
+    { "helper-arg",	required_argument,	NULL, 'G' },
+    { "set-uid",    required_argument,  NULL, 'u' },
+    { "set-gid",    required_argument,  NULL, 'g' },
     { "once-only", 	no_argument,       	NULL, 'o' },
     { "help",      	no_argument,       	NULL, 'h' },
     { "version",   	no_argument,       	NULL, 'V' }
 };
 #endif
 
+static    char	handler[MAX_PATH];
+static    char	handler_arg[MAX_PATH];
+static    char	filename[MAX_PATH];
+static    int   use_handler =0;
+static    int   use_handler_arg =0;
+static    int   i_am_handler =0;
+
 /* Main function.
  */
 int
@@ -155,11 +199,16 @@
     int		use_american_date_formats = 0;
     char 	read_buf[BUFSIZE];
     char 	tzbuf[BUFSIZE];
-    char	filename[MAX_PATH];
     char	*start_time = NULL;
     char	*template;
     char	*linkname = NULL;
     char	*prevlinkname = NULL;
+#ifndef _WIN32
+    uid_t	new_uid = 0;
+    gid_t	new_gid = 0;
+    int		change_uid = 0;
+    int		change_gid = 0;
+#endif
     mode_t	linktype = 0;
     int 	n_bytes_read;
     int		ch;
@@ -167,6 +216,10 @@
     time_t	time_offset = 0;
     time_t	next_period = 0;
     int 	log_fd = -1;
+    
+    memset( handler, '\0', MAX_PATH );
+    memset( handler_arg, '\0', MAX_PATH );
+    memset( filename, '\0', MAX_PATH );
 
 #ifndef _WIN32
     while ((ch = getopt_long(argc, argv, short_options, long_options, NULL)) != EOF)
@@ -234,6 +287,16 @@
 	    }		
 	    break;
 	    
+#ifndef _WIN32
+	case 'u':
+	    new_uid = parse_uid(optarg, argv[0]);
+	    change_uid = 1;
+	    break;
+	case 'g':
+	    new_gid = parse_gid(optarg, argv[0]);
+	    change_gid = 1;
+	    break;
+#endif
 	case 'o':
 	    periodicity = ONCE_ONLY;
 	    break;
@@ -248,7 +311,15 @@
 		debug_file = fopen(optarg, "a+");
 	    }
 	    break;
-	    
+	case 'r':
+            strncat(handler, optarg, MAX_PATH );
+            use_handler=1;
+            break;
+	case 'G':
+            strncat(handler_arg, optarg, MAX_PATH );
+            use_handler_arg=1;
+            break;
+
 	case 'V':
 	    fprintf(stderr, VERSION_MSG);
 	    exit(0);
@@ -266,6 +337,17 @@
 	exit(1);
     }
 
+#ifndef _WIN32
+    if (change_gid && setgid(new_gid) == -1) {
+	fprintf(stderr, "setgid: unable to change to gid: %d\n", new_gid);
+       	exit(1);
+    }
+    if (change_uid && setuid(new_uid) == -1) {
+	fprintf(stderr, "setuid: unable to change to uid: %d\n", new_uid);
+       	exit(1);
+    }
+#endif
+
     DEBUG((VERSION_MSG "\n"));
 
     if (start_time)
@@ -306,6 +388,10 @@
     DEBUG(("Rotation period is per %d %s\n", period_multiple, periods[periodicity]));
 
 
+#ifndef _WIN32
+    set_signal_handlers();
+#endif
+
     /* Loop, waiting for data on standard input */
 
     for (;;)
@@ -316,15 +402,17 @@
 	n_bytes_read = read(0, read_buf, sizeof read_buf);
 	if (n_bytes_read == 0)
 	{
-	    exit(3);
+	    cleanup(3);
 	}
 	if (errno == EINTR)
 	{
-	    continue;
+	    /* 
+             * fall through, it may have been alarm, in which case it will be time to rotate.
+             * */
 	}
 	else if (n_bytes_read < 0)
 	{
-	    exit(4);
+	    cleanup(4);
 	}
 
 	time_now = time(NULL) + time_offset;
@@ -336,6 +424,7 @@
 	{
 	    close(log_fd);
 	    log_fd = -1;
+            fork_to_handle_file();
 	}
 	
 	/* If there is no log file open then open a new one.
@@ -345,6 +434,7 @@
 	    log_fd = new_log_file(template, linkname, linktype, prevlinkname,
 				  periodicity, period_multiple, period_delay,
 				  filename, sizeof (filename), time_now, &next_period);
+            alarm( next_period - time_now );
 	}
 
 	DEBUG(("%s (%d): wrote message; next period starts at %s (%d) in %d secs\n",
@@ -354,10 +444,10 @@
 
 	/* Write out the log data to the current log file.
 	 */
-	if (write(log_fd, read_buf, n_bytes_read) != n_bytes_read)
+	if (n_bytes_read && write(log_fd, read_buf, n_bytes_read) != n_bytes_read)
 	{
 	    perror(filename);
-	    exit(5);
+	    cleanup(5);
 	}
     }
 
@@ -383,6 +473,7 @@
     struct tm 	*tm;
     int 	log_fd;
 
+
     start_of_period = start_of_this_period(time_now, periodicity, period_multiple);
     tm = localtime(&start_of_period);
     strftime(pfilename, BUFSIZE, template, tm);
@@ -394,13 +485,13 @@
 	   timestamp(*pnext_period), *pnext_period,
 	   *pnext_period - time_now));
     
-    log_fd = open(pfilename, O_WRONLY|O_CREAT|O_APPEND, FILE_MODE);
+    log_fd = openwrapper(pfilename);
     
 #ifndef DONT_CREATE_SUBDIRS
     if ((log_fd < 0) && (errno == ENOENT))
     {
 	create_subdirs(pfilename);
-	log_fd = open(pfilename, O_WRONLY|O_CREAT|O_APPEND, FILE_MODE);
+	log_fd = openwrapper(pfilename);
     }
 #endif	    
 
@@ -416,3 +507,179 @@
     }
     return log_fd;
 }
+
+/* 
+ * fork, then exec an external handler to deal with rotated file.
+ */
+void
+fork_to_handle_file()
+{
+    int fk ;
+    static int childpid=0;
+
+    if( ! use_handler || !i_am_handler || handler[0] =='\0' || filename[0] == '\0' )
+    {
+        return;
+    }
+    fk=fork();
+    if( fk < 0 )
+    {
+        perror("couldnt fork");
+        exit(2);
+    }else if( fk > 0 )
+    {
+        if( childpid )
+        {
+            /* 
+             * collect zombies. run twice, in case one or more children took longer than
+             * the rotation period for a while, this will eventually clean them up.
+             * Of course, if handler children take longer than rotation period to handle
+             * things, you will eventually have a big problem.
+             * 
+             * */ 
+            (void) waitpid( 0, NULL, WNOHANG | WUNTRACED );
+            (void) waitpid( 0, NULL, WNOHANG | WUNTRACED );
+        }
+        childpid=fk;
+        return; /* parent */
+    }
+    /* child */
+    /* dont muck with stdin or out of parent, but allow stderr to be commingled */
+    close(0);
+    close(1);
+    handle_file();
+}
+
+/* 
+ * exec an external handler to deal with rotated file.
+ */
+void 
+handle_file()
+{
+    char **exec_argv ;
+    if( ! use_handler || !i_am_handler || handler[0] =='\0' || filename[0] == '\0' )
+    {
+        return;
+    }
+    if ( use_handler_arg == 0 )
+    { 
+        exec_argv = malloc( sizeof( char *)*3);
+        exec_argv[0] = strdup( handler );
+        exec_argv[1] = strdup( filename );
+        exec_argv[2] = NULL;
+    }else
+    {
+        exec_argv = malloc( sizeof( char *)*4);
+        exec_argv[0] = strdup( handler );
+        exec_argv[1] = strdup( handler_arg );
+        exec_argv[2] = strdup( filename );
+        exec_argv[3] = NULL ;
+    }
+    execvp( exec_argv[0], exec_argv );
+    perror("cant execvp");
+    exit(2);
+}
+
+
+
+#ifndef _WIN32
+/* 
+ * wrapper to be called as signal handler.
+ */
+void 
+handle_file_on_sig( int sig, siginfo_t *si, void *v )
+{
+    handle_file();
+    /* not reached */
+    exit( 3 );
+};
+
+/* 
+ * wrapper to be called for alarm signal
+ */
+void 
+alarm_signal_handler( int sig, siginfo_t *si, void *v )
+{
+        ;
+        /* 
+         * do nothing; the key thing is that the alarm will cause the read()
+         * to fail with errno=EINTR. this empty handler is required, because the
+         * default handler will exit(1)
+         *
+         */
+};
+
+void
+set_signal_handlers()
+{
+    /* 
+     * all signals which usually kill a process that can be caught are
+     * set to handle_file when received. This will make apache shutdowns more 
+     * graceful even if use_handler is false.
+     */
+    setsig_handler( SIGHUP, handle_file_on_sig );
+    setsig_handler( SIGINT, handle_file_on_sig );
+    setsig_handler( SIGQUIT, handle_file_on_sig );
+    setsig_handler( SIGILL, handle_file_on_sig );
+    setsig_handler( SIGABRT, handle_file_on_sig );
+    setsig_handler( SIGBUS, handle_file_on_sig );
+    setsig_handler( SIGFPE, handle_file_on_sig );
+    setsig_handler( SIGPIPE, handle_file_on_sig );
+    setsig_handler( SIGTERM, handle_file_on_sig );
+    setsig_handler( SIGUSR1, handle_file_on_sig );
+
+    /* sigalrm is used to break out of read() when it is time to rotate the log. */
+    setsig_handler( SIGALRM, alarm_signal_handler );
+}
+
+void
+setsig_handler( int signum,  void (*action)(int, siginfo_t *, void *))
+{
+    struct sigaction siga ;
+    memset( &siga, '\0', sizeof( struct sigaction ));
+    siga.sa_sigaction= action ;
+    siga.sa_flags = SA_SIGINFO ;
+    if( -1== sigaction( signum, &siga, NULL ))
+    {
+        perror( "cant set sigaction" );
+    }
+}
+#endif
+
+
+/* 
+ * cleanup
+ */
+void
+cleanup( int exit_status )
+{
+    handle_file();
+    exit(exit_status);
+}
+
+/* 
+ * only the first cronolog process to open a particular file is responsible
+ * for starting the cleanup process later. This wrapper sets i_am_handler
+ * according to that logic.
+ * */
+int 
+openwrapper( const char *ofilename )
+{
+    int ret;
+    if( use_handler !=1 )
+    {
+        return open(ofilename, OPEN_SHARED, S_IRWXU );
+    }
+    ret = open(ofilename, OPEN_EXCLUSIVE, S_IRWXU );
+    if( ret < 0 )
+    {
+        ret = open(ofilename, OPEN_SHARED, S_IRWXU );
+        i_am_handler= 0;
+    }
+    else
+    {
+        i_am_handler=1;
+    }
+    return ret;
+}
+
diff -Nur cronolog-1.6.2/src/cronoutils.c cronolog-1.6.2-jumbo-20031008/src/cronoutils.c
--- cronolog-1.6.2/src/cronoutils.c	2001-05-03 12:43:21.000000000 -0400
+++ cronolog-1.6.2-jumbo-20031008/src/cronoutils.c	2003-10-08 00:22:10.000000000 -0400
@@ -195,11 +195,11 @@
 {
     struct stat		stat_buf;
     
-    if (stat(prevlinkname, &stat_buf) == 0)
+    if (lstat(prevlinkname, &stat_buf) == 0)
     {
 	unlink(prevlinkname);
     }
-    if (stat(linkname, &stat_buf) == 0)
+    if (lstat(linkname, &stat_buf) == 0)
     {
 	if (prevlinkname) {
 	    rename(linkname, prevlinkname);
@@ -710,4 +710,50 @@
     return retval;
 }
 
-    
+
+#ifndef _WIN32
+/* Turn a string specifying either a username or UID into an actual
+ * uid_t for use in setuid(). A string is assumed to be a UID if
+ * it contains only decimal digits. */
+uid_t
+parse_uid(char *user, char *argv0)
+{
+    char		*probe = user;
+    struct passwd	*ent;
+
+    while (*probe && isdigit(*probe)) {
+	probe++;
+    }
+    if (!(*probe)) {
+	return atoi(user);
+    }
+    if (!(ent = getpwnam(user))) {
+	fprintf(stderr, "%s: Bad username %s\n", argv0, user);
+	exit(1);
+    }
+    return (ent->pw_uid);
+}
+
+
+/* Turn a string specifying either a group name or GID into an actual
+ * gid_t for use in setgid(). A string is assumed to be a GID if
+ * it contains only decimal digits. */
+gid_t
+parse_gid(char *group, char *argv0)
+{
+    char		*probe = group;
+    struct group	*ent;
+
+    while (*probe && isdigit(*probe)) {
+	probe++;
+    }
+    if (!(*probe)) {
+	return atoi(group);
+    }
+    if (!(ent = getgrnam(group))) {
+	fprintf(stderr, "%s: Bad group name %s\n", argv0, group);
+	exit(1);
+    }
+    return (ent->gr_gid);
+}
+#endif /* _WIN32 */
diff -Nur cronolog-1.6.2/src/cronoutils.h cronolog-1.6.2-jumbo-20031008/src/cronoutils.h
--- cronolog-1.6.2/src/cronoutils.h	2001-05-03 12:40:12.000000000 -0400
+++ cronolog-1.6.2-jumbo-20031008/src/cronoutils.h	2003-10-08 00:22:10.000000000 -0400
@@ -84,6 +84,8 @@
 #include <limits.h>
 #ifndef _WIN32
 #include <unistd.h>
+#include <pwd.h>
+#include <grp.h>
 #else
 #include <io.h>
 #include <direct.h>
@@ -172,7 +174,8 @@
 void		print_debug_msg(char *msg, ...);
 time_t		parse_time(char *time_str, int);
 char 		*timestamp(time_t thetime);
-
+uid_t		parse_uid(char *user, char *argv0);
+gid_t		parse_gid(char *group, char *argv0);
 
 /* Global variables */
 
diff -Nur cronolog-1.6.2/src/zip_send_rm.sh cronolog-1.6.2-jumbo-20031008/src/zip_send_rm.sh
--- cronolog-1.6.2/src/zip_send_rm.sh	1969-12-31 19:00:00.000000000 -0500
+++ cronolog-1.6.2-jumbo-20031008/src/zip_send_rm.sh	2003-10-10 12:37:22.000000000 -0400
@@ -0,0 +1,64 @@
+#!/bin/bash
+## ----------------------------------------------------------------------
+## ----------------------------------------------------------------------
+##
+## File:      zip_send_rm
+## Author:    mgrosso 
+## Created:   Fri Oct  3 18:18:38 EDT 2003 on dhcp-172-18-102-101.looksmart.com
+## Project:   apache, cronolog
+## Purpose:   make damn sure the log file arrives at its destinations
+## 
+## Copyright (c) 2003 LookSmart. All Rights Reserved.
+## 
+## $Id$
+## ----------------------------------------------------------------------
+## ----------------------------------------------------------------------
+
+
+function do_or_die()
+{
+    $@
+    if [ $? -ne 0 ] ; then 
+        logger -s -p local0.crit "$0 puking on [ $@ ]"
+        exit 1
+    fi
+}
+
+if [ $# == 2 ] ; then
+    SCPDESTLIST=$1
+    shift
+fi
+FILE=$1
+
+
+
+do_or_die gzip $FILE
+
+if [ -z $SCPDESTLIST ] ; then
+    exit 0
+fi
+
+FILE=${FILE}.gz
+
+# spawn subshells, so that each scp can suceed or fail on its own.
+# while this is less efficient from a bandwidth and cpu perspective, it 
+# is safer because it ensures that a hanging dest cant prevent the other 
+# dests from working.
+
+SCPDESTINATIONS=$( echo $SCPDESTLIST | sed -e 's/,/ /g' )
+for DEST in $SCPDESTINATIONS ; do
+    (
+        HDEST=$( echo $DEST | cut -d ':' -f 1 )
+        BACKUP_LINK=${HDEST}-$FILE
+        do_or_die ln $FILE $BACKUP_LINK
+        do_or_die scp $FILE $DEST
+        do_or_die rm $BACKUP_LINK
+    ) &
+done
+wait
+# we dont delete the file unless it arrive at all destinations
+# because if one destination failed, then ${DEST}-$FILE will still exist
+# even after we rm $FILE
+do_or_die rm $FILE
+exit 0
+





-- 
#######################################################################
# Matt Grosso           cell    (201)780-9592   mgrosso@looksmart.net #
# work ny (212)324-1904 fax ny  (212)324-1910   work sf (415)348-7756 #
# pgp public key is at http://www.falconweb.com/~mattg/publickey.pgp  #
#######################################################################