2 webalizer - a web server log analysis program
4 Copyright (C) 1997-2011 Bradford L. Barrett
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version, and provided that the above
10 copyright and permission notice is included with all distributed
11 copies of this or derived software.
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
24 /*********************************************/
25 /* STANDARD INCLUDES */
26 /*********************************************/
33 #include <unistd.h> /* normal stuff */
35 #include <sys/utsname.h>
38 /* ensure sys/types */
40 #include <sys/types.h>
43 /* Need socket header? */
44 #ifdef HAVE_SYS_SOCKET_H
45 #include <sys/socket.h>
48 /* some systems need this */
53 #ifdef USE_DNS /* skip everything in this file if no DNS */
55 #include <netinet/in.h> /* include stuff we need for dns lookups, */
56 #include <arpa/inet.h> /* DB access, file control, etc... */
58 #include <netdb.h> /* ensure getaddrinfo/getnameinfo */
60 #include <sys/signal.h>
64 #include <db.h> /* DB header ****************/
65 #include "webalizer.h" /* main header */
66 #include "lang.h" /* language declares */
67 #include "hashtab.h" /* hash table functions */
68 #include "parser.h" /* log parser functions */
69 #include "dns_resolv.h" /* our header */
73 DB
*dns_db
= NULL
; /* DNS cache database */
76 DB
*geo_db
= NULL
; /* GeoDB database */
77 DBC
*geo_dbc
= NULL
; /* GeoDB database cursor */
79 struct dns_child child
[MAXCHILD
]; /* DNS child pipe data */
81 DNODEPTR host_table
[MAXHASH
]; /* hostname/ip hash table */
83 char buffer
[BUFSIZE
]; /* log file record buffer */
84 char tmp_buf
[BUFSIZE
]; /* used to temp save above */
85 struct utsname system_info
; /* system info structure */
87 int raiseSigChild
= 1;
90 time_t start_time
, end_time
;
93 extern char *our_gzgets(void *, char *, int); /* external our_gzgets func */
95 /* internal function prototypes */
97 static void process_list(DNODEPTR
);
98 static void sigChild(int);
99 static void db_put(char *, char *, int);
100 void set_fl(int, int);
101 void clr_fl(int, int);
102 int iptype(char *, unsigned char *);
104 /*********************************************/
105 /* RESOLVE_DNS - lookup IP in cache */
106 /*********************************************/
108 void resolve_dns(struct log_struct
*log_rec
)
112 /* aligned dnsRecord to prevent Solaris from doing a dump */
113 /* (not found in debugger, as it can dereference it :( */
114 struct dnsRecord alignedRecord
;
116 if (!dns_db
) return; /* ensure we have a dns db */
118 memset(&query
, 0, sizeof(query
));
119 memset(&response
, 0, sizeof(response
));
120 query
.data
= log_rec
->hostname
;
121 query
.size
= strlen(log_rec
->hostname
);
123 if (debug_mode
) fprintf(stderr
,"Checking %s...", log_rec
->hostname
);
125 if ( (i
=dns_db
->get(dns_db
, NULL
, &query
, &response
, 0)) == 0)
127 memcpy(&alignedRecord
, response
.data
, sizeof(struct dnsRecord
));
128 strncpy (log_rec
->hostname
,
129 ((struct dnsRecord
*)response
.data
)->hostName
,
131 log_rec
->hostname
[MAXHOST
-1]=0;
133 fprintf(stderr
," found: %s (%ld)\n",
134 log_rec
->hostname
, alignedRecord
.timeStamp
);
136 else /* not found or error occured during get */
140 if (i
==DB_NOTFOUND
) fprintf(stderr
," not found\n");
141 else fprintf(stderr
," error (%d)\n",i
);
146 /*********************************************/
147 /* DNS_RESOLVER - read log and lookup IP's */
148 /*********************************************/
150 int dns_resolver(void *log_fp
)
153 DNODEPTR l_list
= NULL
;
156 int save_verbose
=verbose
;
158 u_int64_t listEntries
= 0;
160 struct sigaction sigPipeAction
;
162 /* aligned dnsRecord to prevent Solaris from doing a dump */
163 /* (not found in debugger, as it can dereference it :( */
164 struct dnsRecord alignedRecord
;
166 struct flock tmp_flock
;
168 tmp_flock
.l_whence
=SEEK_SET
; /* default flock fields */
175 /* get processing start time */
176 start_time
= time(NULL
);
178 /* minimal sanity check on it */
179 if(stat(dns_cache
, &dbStat
) < 0)
184 dns_db
=NULL
; return 0; /* disable cache */
189 if(!dbStat
.st_size
) /* bogus file, probably from a crash */
191 unlink(dns_cache
); /* remove it so we can recreate... */
195 /* open cache file */
196 if ( (db_create(&dns_db
, NULL
, 0) != 0) ||
197 (dns_db
->open(dns_db
, NULL
,
198 dns_cache
, NULL
, DB_HASH
,
199 DB_CREATE
, 0644) != 0) )
201 /* Error: Unable to open DNS cache file <filename> */
202 if (verbose
) fprintf(stderr
,"%s %s\n",msg_dns_nodb
,dns_cache
);
205 return 0; /* disable cache */
208 /* get file descriptor */
209 dns_db
->fd(dns_db
, &dns_fd
);
211 tmp_flock
.l_type
=F_WRLCK
; /* set read/write lock type */
212 if (fcntl(dns_fd
,F_SETLK
,&tmp_flock
) < 0) /* and barf if we cant lock */
214 /* Error: Unable to lock DNS cache file <filename> */
215 if (verbose
) fprintf(stderr
,"%s %s\n",msg_dns_nolk
,dns_cache
);
216 dns_db
->close(dns_db
, 0);
219 return 0; /* disable cache */
222 /* Setup signal handlers */
223 sigPipeAction
.sa_handler
= SIG_IGN
;
224 sigPipeAction
.sa_flags
= SA_RESTART
;
225 sigemptyset(&sigPipeAction
.sa_mask
);
227 sigaction(SIGPIPE
, &sigPipeAction
, NULL
);
229 /* disable warnings/errors for this run... */
232 /* Main loop to read log records (either regular or zipped) */
233 while ( (gz_log
)?(our_gzgets((void *)log_fp
,buffer
,BUFSIZE
) != Z_NULL
):
234 (fgets(buffer
,BUFSIZE
,log_fname
?(FILE *)log_fp
:stdin
) != NULL
))
236 if (strlen(buffer
) == (BUFSIZE
-1))
238 /* get the rest of the record */
239 while ( (gz_log
)?(our_gzgets((void *)log_fp
,buffer
,BUFSIZE
)!=Z_NULL
):
240 (fgets(buffer
,BUFSIZE
,log_fname
?(FILE *)log_fp
:stdin
)!=NULL
))
242 if (strlen(buffer
) < BUFSIZE
-1) break;
244 continue; /* go get next record if any */
247 strcpy(tmp_buf
, buffer
); /* save buffer in case of error */
248 if(parse_record(buffer
)) /* parse the record */
250 struct addrinfo hints
, *ares
;
251 memset(&hints
, 0, sizeof(hints
));
252 hints
.ai_family
= AF_UNSPEC
;
253 hints
.ai_socktype
= SOCK_STREAM
;
254 hints
.ai_flags
= AI_NUMERICHOST
;
255 if (0 == getaddrinfo(log_rec
.hostname
, "0", &hints
, &ares
))
258 memset(&q
, 0, sizeof(q
));
259 memset(&r
, 0, sizeof(r
));
260 q
.data
= log_rec
.hostname
;
261 q
.size
= strlen(log_rec
.hostname
);
263 /* Check if we have it in DB */
264 if ( (i
=dns_db
->get(dns_db
, NULL
, &q
, &r
, 0)) == 0 )
266 /* have a record for this address */
267 memcpy(&alignedRecord
, r
.data
, sizeof(struct dnsRecord
));
268 if (alignedRecord
.timeStamp
!= 0)
269 /* If it's not permanent, check if it's TTL has expired */
270 if ( (runtime
-alignedRecord
.timeStamp
) > (86400*cache_ttl
) )
271 put_dnode(log_rec
.hostname
, ares
->ai_addr
,
272 ares
->ai_addrlen
, host_table
);
277 put_dnode(log_rec
.hostname
, ares
->ai_addr
,
278 ares
->ai_addrlen
, host_table
);
284 verbose
= save_verbose
; /* restore verbosity level... */
288 /* build our linked list l_list */
289 for(i
=0;i
< MAXHASH
; i
++)
291 for(h_entries
=host_table
[i
]; h_entries
; h_entries
= h_entries
->next
)
293 h_entries
->llist
= l_list
;
301 /* No valid addresses found... */
302 if (verbose
>1) printf("%s\n",msg_dns_none
);
303 tmp_flock
.l_type
=F_UNLCK
;
304 fcntl(dns_fd
, F_SETLK
, &tmp_flock
);
305 dns_db
->close(dns_db
, 0);
309 /* process our list now... */
310 process_list(l_list
);
312 /* get processing end time */
313 end_time
= time(NULL
);
315 /* display DNS processing statistics */
316 if (time_me
|| (verbose
>1))
318 if (verbose
<2 && time_me
) printf("DNS: ");
319 printf("%llu %s ",listEntries
, msg_addresses
);
321 /* total processing time in seconds */
322 temp_time
= difftime(end_time
,start_time
);
323 if (temp_time
==0) temp_time
=1;
324 printf("%s %.0f %s", msg_in
, temp_time
, msg_seconds
);
326 /* calculate records per second */
328 i
=( (int)((float)listEntries
/temp_time
) );
331 if ( (i
>0) && (i
<=listEntries
) ) printf(", %d/sec\n", i
);
335 /* processing done, exit */
336 tmp_flock
.l_type
=F_UNLCK
;
337 fcntl(dns_fd
, F_SETLK
, &tmp_flock
);
338 dns_db
->close(dns_db
, 0);
343 /*********************************************/
344 /* PROCESS_LIST - do the resoluton... */
345 /*********************************************/
347 static void process_list(DNODEPTR l_list
)
351 char child_buf
[MAXHOST
+3-((unsigned long)&trav
+sizeof(trav
))%3];
352 char dns_buf
[MAXHOST
];
355 int nof_children
= 0;
357 char hbuf
[NI_MAXHOST
];
359 struct sigaction sigChildAction
;
361 sigChildAction
.sa_handler
= sigChild
;
362 sigChildAction
.sa_flags
= SA_NOCLDSTOP
|SA_RESTART
;
363 sigemptyset(&sigChildAction
.sa_mask
);
367 sigaction(SIGCHLD
, &sigChildAction
, NULL
);
369 /* fire up our child processes */
370 for(i
=0; i
< dns_children
; i
++)
372 if(pipe(child
[i
].inpipe
))
374 if (verbose
) fprintf(stderr
,"INPIPE creation error");
375 return; /* exit(1) */
378 if(pipe(child
[i
].outpipe
))
380 if (verbose
) fprintf(stderr
,"OUTPIPE creation error");
381 return; /* exit(1); */
389 if (verbose
) fprintf(stderr
,"FORK error");
390 return; /* exit(1); */
397 close(child
[i
].inpipe
[0]);
398 close(child
[i
].outpipe
[1]);
400 /* get struct sockaddr_storage here */
401 while((size
= read(child
[i
].outpipe
[0], child_buf
, MAXHOST
)))
405 perror("read error");
410 /* Clear out our buffer */
411 memset(hbuf
,0,NI_MAXHOST
);
413 if(0 == getnameinfo((struct sockaddr
*)child_buf
, size
,
414 hbuf
, sizeof(hbuf
), NULL
, 0, NI_NAMEREQD
))
416 /* must be at least 4 chars */
419 /* If long hostname, take max domain name part */
420 if ((size
= strlen(hbuf
)) > MAXHOST
-2)
421 strcpy(child_buf
,(hbuf
+(size
-MAXHOST
+1)));
422 else strcpy(child_buf
, hbuf
);
423 size
= strlen(child_buf
);
428 printf("Child %d getnameinfo bad hbuf!\n",i
);
434 printf("Child %d getnameinfo failed!\n",i
);
437 if (write(child
[i
].inpipe
[1], child_buf
, size
) == -1)
439 perror("write error");
444 close(child
[i
].inpipe
[1]);
445 close(child
[i
].outpipe
[0]);
448 printf( "Child %d got closed input, shutting down\n", i
);
457 child
[i
].flags
= DNS_CHILD_READY
|DNS_CHILD_RUNNING
;
459 close(child
[i
].inpipe
[1]);
460 close(child
[i
].outpipe
[0]);
462 set_fl(child
[i
].inpipe
[0], O_NONBLOCK
);
471 static struct timeval selectTimeval
;
482 while((pid
= waitpid(-1, NULL
, WNOHANG
)) > 0)
484 for(i
=0;i
<dns_children
;i
++)
486 if(child
[i
].pid
== pid
)
489 child
[i
].flags
&= ~(DNS_CHILD_READY
|DNS_CHILD_RUNNING
);
493 printf("Reaped Child %d\n", pid
);
500 continue; /* while, nof children has just changed */
503 for(i
=0;i
<dns_children
;i
++)
505 if(child
[i
].flags
& DNS_CHILD_RUNNING
) /* Child is running */
507 if(child
[i
].flags
& DNS_CHILD_READY
)
509 child
[i
].flags
&= ~DNS_CHILD_READY
;
511 if(trav
) /* something to resolve */
513 if (write(child
[i
].outpipe
[1], &trav
->addr
,
514 trav
->addrlen
) != -1)
516 /* We will watch this child */
518 FD_SET(child
[i
].inpipe
[0], &rd_set
);
519 max_fd
= MAX(max_fd
, child
[i
].inpipe
[0]);
522 printf("Giving %d bytes to Child %d\n",
527 else /* write error */
529 if(errno
!= EINTR
) /* Could be a signal */
531 perror("Could not write to pipe");
532 close(child
[i
].outpipe
[1]); /* kill */
533 child
[i
].flags
&= ~DNS_CHILD_RUNNING
; /* child */
537 else /* List is complete */
539 close(child
[i
].outpipe
[1]); /* Go away */
540 child
[i
].flags
&= ~DNS_CHILD_RUNNING
; /* Child is dead */
545 /* Look, the busy child... */
546 FD_SET(child
[i
].inpipe
[0], &rd_set
);
547 max_fd
= MAX(max_fd
, child
[i
].inpipe
[0]);
552 selectTimeval
.tv_sec
= 5; /* This stuff ticks in 5 second intervals */
553 selectTimeval
.tv_usec
= 0;
555 switch(res
= select(max_fd
+1, &rd_set
, NULL
, NULL
, &selectTimeval
))
559 if(errno
!= EINTR
) /* Could be a signal */
560 perror("Error in select");
565 case 0: /* Timeout, just fall once through the child loop */
575 for(i
=0; i
< dns_children
;i
++)
577 if(!res
) /* All file descriptors done */
580 if(FD_ISSET(child
[i
].inpipe
[0], &rd_set
))
584 res
--; /* One less... */
586 switch (size
=read(child
[i
].inpipe
[0], dns_buf
, MAXHOST
))
591 perror("Could not read from pipe");
596 /* EOF. Child has closed Pipe. It shouldn't have */
597 /* done that, could be an error or something. */
599 close(child
[i
].outpipe
[1]);
600 child
[i
].flags
&= ~DNS_CHILD_RUNNING
;
603 printf("Child %d wants to be reaped\n", i
);
610 dns_buf
[size
] = '\0';
611 if( strlen(dns_buf
) > 1 &&
612 memcmp(dns_buf
, &(child
[i
].cur
->addr
),
613 sizeof(child
[i
].cur
->addr
)))
616 printf("Child %d Got a result: %s -> %s\n",
617 i
, child
[i
].cur
->string
, dns_buf
);
618 db_put(child
[i
].cur
->string
, dns_buf
, 0);
623 printf("Child %d could not resolve: %s (%s)\n",
624 i
, child
[i
].cur
->string
,
625 (cache_ips
)?"cache":"no cache");
626 if (cache_ips
) /* Cache non-resolved? */
627 db_put(child
[i
].cur
->string
,
628 child
[i
].cur
->string
,1);
632 printf("Child %d back in task pool\n", i
);
634 /* Child is back in the task pool */
635 child
[i
].flags
|= DNS_CHILD_READY
;
648 /*********************************************/
649 /* SET_FL - set flag on pipe FD */
650 /*********************************************/
652 void set_fl(int fd
, int flags
)
656 /* get current flags */
657 if ((val
=fcntl(fd
, F_GETFL
, 0)) < 0)
658 if (verbose
) fprintf(stderr
,"set_fl F_GETFL error\n");
663 /* and write them back */
664 if ((val
=fcntl(fd
, F_SETFL
, val
)) < 0)
665 if (verbose
) fprintf(stderr
,"set_fl F_SETFL error\n");
668 /*********************************************/
669 /* CLR_FL - clear flag on pipe FD */
670 /*********************************************/
672 void clr_fl(int fd
, int flags
)
676 /* Get current flags */
677 if ((val
=fcntl(fd
, F_GETFL
, 0)) < 0)
678 if (verbose
) fprintf(stderr
,"clr_fl F_GETFL error\n");
683 /* and write them back */
684 if ((val
=fcntl(fd
, F_SETFL
, val
)) < 0)
685 if (verbose
) fprintf(stderr
,"clr_fl F_SETFL error\n");
688 /*********************************************/
689 /* DB_PUT - put key/val in the cache db */
690 /*********************************************/
692 static void db_put(char *key
, char *value
, int numeric
)
696 struct dnsRecord
*recPtr
= NULL
;
697 int nameLen
= strlen(value
)+1;
699 /* Align to multiple of eight bytes */
700 int recSize
= (sizeof(struct dnsRecord
)+nameLen
+7) & ~0x7;
702 /* make sure we have a db ;) */
705 if((recPtr
= calloc(1, recSize
)))
707 recPtr
->timeStamp
= runtime
;
708 recPtr
->numeric
= numeric
;
709 memcpy(&recPtr
->hostName
, value
, nameLen
);
710 memset(&k
, 0, sizeof(k
));
711 memset(&v
, 0, sizeof(v
));
713 /* Ensure all data is lowercase */
714 cp
=key
; while (*cp
++!='\0') *cp
=tolower(*cp
);
715 cp
=value
; while (*cp
++!='\0') *cp
=tolower(*cp
);
718 k
.size
= strlen(key
);
723 if ( dns_db
->put(dns_db
, NULL
, &k
, &v
, 0) != 0 )
724 if (verbose
>1) fprintf(stderr
,"db_put fail!\n");
730 /*********************************************/
731 /* SIGCHILD - raise our signal */
732 /*********************************************/
734 static void sigChild(int signum
)
739 /*********************************************/
740 /* OPEN_CACHE - open our cache file RDONLY */
741 /*********************************************/
746 struct flock tmp_flock
;
748 tmp_flock
.l_whence
=SEEK_SET
; /* default flock fields */
752 tmp_flock
.l_type
=F_RDLCK
;
754 /* double check filename was specified */
755 if(!dns_cache
) { dns_db
=NULL
; return 0; }
757 /* minimal sanity check on it */
758 if(stat(dns_cache
, &dbStat
) < 0)
760 if(errno
!= ENOENT
) return 0;
764 if(!dbStat
.st_size
) /* bogus file, probably from a crash */
766 unlink(dns_cache
); /* remove it so we can recreate... */
770 /* open cache file */
771 if ( (db_create(&dns_db
, NULL
, 0) != 0) ||
772 (dns_db
->open(dns_db
, NULL
,
773 dns_cache
, NULL
, DB_HASH
,
774 DB_RDONLY
, 0644) != 0) )
776 /* Error: Unable to open DNS cache file <filename> */
777 if (verbose
) fprintf(stderr
,"%s %s\n",msg_dns_nodb
,dns_cache
);
778 return 0; /* disable cache */
781 /* get file descriptor */
782 dns_db
->fd(dns_db
, &dns_fd
);
784 /* Get shared lock on cache file */
785 if (fcntl(dns_fd
, F_SETLK
, &tmp_flock
) < 0)
787 if (verbose
) fprintf(stderr
,"%s %s\n",msg_dns_nolk
,dns_cache
);
788 dns_db
->close(dns_db
, 0);
794 /*********************************************/
795 /* CLOSE_CACHE - close our RDONLY cache */
796 /*********************************************/
800 struct flock tmp_flock
;
802 tmp_flock
.l_whence
=SEEK_SET
; /* default flock fields */
806 tmp_flock
.l_type
=F_UNLCK
;
808 /* clear lock and close cache file */
809 fcntl(dns_fd
, F_SETLK
, &tmp_flock
);
810 dns_db
->close(dns_db
, 0);
814 /*********************************************/
815 /* GEODB_OPEN - Open GeoDB database/cursor */
816 /*********************************************/
818 DB
*geodb_open(char *dbname
)
823 snprintf(buf
,sizeof(buf
),"%s/GeoDB.dat",GEODB_LOC
);
825 strncpy(buf
,dbname
,sizeof(buf
)-1);
826 buf
[sizeof(buf
)-1]='\0';
828 /* create database thingie */
829 if ( db_create(&geo_db
, NULL
, 0) ) return NULL
;
831 /* open the database */
832 if (geo_db
->open(geo_db
,NULL
,buf
,NULL
,DB_BTREE
,DB_RDONLY
,0)) return NULL
;
834 /* create our cursor */
835 if (geo_db
->cursor(geo_db
,NULL
,&geo_dbc
,0))
837 geo_db
->close(geo_db
,0);
840 /* all is well in the world */
844 /*********************************************/
845 /* GEODB_VER - Get database version info */
846 /*********************************************/
848 char *geodb_ver(DB
*db
, char *str
)
854 memset(&x
, 0, sizeof(x
));
855 memset(&k
, 0, sizeof(k
));
856 memset(&v
, 0, sizeof(v
));
860 i
=geo_db
->get(geo_db
, NULL
, &k
, &v
, 0);
862 if (i
) strncpy(str
, "Unknown", 8);
863 else strncpy(str
, v
.data
+3, v
.size
-3);
867 /*********************************************/
868 /* GEODB_GET_CC - Get country code for IP */
869 /*********************************************/
871 char *geodb_get_cc(DB
*db
, char *ip
, char *buf
)
875 unsigned char addr
[16];
877 memset(addr
, 0, sizeof(addr
));
878 strncpy(buf
, "--", 3);
881 if (!iptype(ip
, addr
)) return buf
;
883 /* kludge for IPv6 mapped IPv4 */
884 if (addr
[0]==0 && addr
[1]==0 && addr
[2]==0) { addr
[10]=0; addr
[11]=0; }
886 /* kludge for IPv6 6to4 (RFC3056) */
887 if (addr
[0]==0x20 && addr
[1]==0x02)
889 memcpy(&addr
[12],&addr
[2],4);
893 memset(&k
, 0, sizeof(k
));
894 memset(&v
, 0, sizeof(v
));
898 i
=geo_dbc
->c_get(geo_dbc
, &k
, &v
, DB_SET_RANGE
);
899 if (!i
) memcpy(buf
, v
.data
, 2);
903 /*********************************************/
904 /* GEODB_CLOSE - close GeoDB database */
905 /*********************************************/
907 void geodb_close(DB
*db
)
912 /*********************************************/
913 /* IPTYPE - get IP type and format addr buf */
914 /*********************************************/
916 int iptype(char *ip
, unsigned char *buf
)
918 if (inet_pton(AF_INET6
, ip
, buf
)>0) return 2;
919 if (inet_pton(AF_INET
, ip
, buf
+12)>0) return 1;