#!/usr/bin/env perl

# Full cgi example - log file analysis

use Time::Local;

# Version 0.2 - Analysis of log files

initvars();
%form = collect_form();
$limitrecs = 500; $nrec = 0;

open (FH,$source);

# Search through all records in the data file that's being analysed

while ($line = <FH>) {
        %info = get_ncsaparts($line);
        next unless ($info{status}) ;

# If a selection form has been completed, look for records we need

        if ($form{select} or $form{host}) {
                $selected = 1;
                foreach $field (@names) {
                        $form{$field} and
                                $info{$field} !~ /$form{$field}/i and
                                        $selected = 0;
                        }
                if ($selected) {
                        $nrec++;
                        if ($nrec <= $limitrecs) {
                                my $thisline;
                                $nexttime = $info{when};
                                $info{when} = $info{when} - $previoustime;
                                $previoustime = $nexttime;
                                foreach $field("when",@names) {
                                        ($form{"_$field"} or ! $form{select})
                                                 and $thisline .= $info{$field}." ";
                                        }
                                $stuff .= "$thisline<br>";
                        }
                        $visitors{$info{host}}++;
                }

# If no selection form has been completed, summarise all hosts

        } else {
                $summaryhost = $info{host};
                if ($summaryhost =~ /[a-z]$/) {
                        $summaryhost =~ s/^[^.]+/xxx/;
                } else {
                        $summaryhost =~ s/\.\d+$/.xxx/;
                }
                $nrec++;
                $visitors{$summaryhost}++;
                }
        }

# All records read. Report on the number of matches and any truncation

$stuff .= ($nrec>=$limitrecs and ($form{select} or $form{host})) ?
        "REPORT TRUNCATED TO $limitrecs out of $nrec<br>":
        "Reported on $nrec records<br>";

# Set fields for display on next form if necessary

unless ($form{select}) {
        foreach $field(@show) {
                $form{"_$field"} = 1;
        }
}

($me) = ($0 =~ m!.*/(.*)!);
$fill{myname} = $me;
$fill{source} = $source;

# Make up a table of all hosts in case user wants to select by host

$stuff .= "<table>";
foreach $v (sort {reverse($a) cmp reverse($b)} keys %visitors) {
        $v1 = $v;
        $v1 =~ s/xxx//;
        $stuff .= "<tr><td><a href=$me?host=$v1>$v</a></td>".
                "<td>$visitors{$v}</td></tr>";
        }
$fill{stuff} = "$stuff</table>";

# Make up form through which to offer the user next options

foreach $field (@names,"when") {
        $checked = $form{"_$field"} ? " CHECKED" : "";
        $fill{formbody} .= "<tr><td>limit $field to</td><td>".
                        "<input name=$field value=\"$form{$field}\"></td>".
                        "<td><input name=_$field type=checkbox$checked> display</td>".
                        "</tr>";
        }

# Read and complete template, send it out to browser

open (FH,"nice.htp");
read (FH,$html,-s "nice.htp");
$html =~ s/%(\w+)%/$fill{$1}/g;

print ("content-type: text/html\n\n$html");

##################################################################

sub get_ncsaparts {
        my ($inline) = @_;
        my %record;
# Extract parts from an NCSA extended log file record (format is next 4 lines)
# 202.187.80.126 - - [09/Feb/2003:00:37:31 -0800]
# "GET /forum/3935408201.html HTTP/1.1" 200 7250
# "http://www.google.com/search?hl=en&ie=UTF-8&oe=UTF-8&q=text+file+and+php"
# "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
        if (my @parts = ($inline =~
                /^(\S+) # IP Address or host name
                \s+\S+\s+\S+\s+ # Ignore rarely used security flds
                \[(\d+)\/(\w+)\/(\d+): # Date
                (\d+):(\d+):(\d+) # Time
                \s+([-+]?\d\d)\d\d\]\s+ # time offset
                "(\w+)\s+(\S+) # method and URL
                \s+\S+"\s+ # Ignore HTTP level
                (\d+)\s+([-0-9]+)\s+ # Status and size
                "(\S+)"\s+ # Referer
                "(.+)"\s*$ # Browser
                /x)) {
        for (my $k=0; $k<@names; $k++) {
                $record{$names[$k]} = $parts[$k];
                }
                $record{when} = timegm($record{second},
                        $record{minute},$record{hour},
                        $record{day},$mnames{$record{month}},
                        $record{year}%100) - $record{houroff} * 3600;
        } else {
                $record{status} = 0;
        }
        return %record;
        }

sub collect_form {
        my %ret;
        if ($ENV{REQUEST_METHOD} eq "POST") {
                read (STDIN,$qs,$ENV{CONTENT_LENGTH});
                $ret{rqm} = "POST";
        } else {
                $qs = $ENV{QUERY_STRING};
                $ret{rqm} = "GET";
                }
        my @els = split(/&/,$qs);
        foreach (@els) {
                my ($nam,$val) = split(/=/);
                $val =~ tr/+/ /;
                $val =~ s/%(..)/pack("C",hex($1))/ge;
                $ret{$nam} = $val;
                }
        return %ret;
        }

sub initvars {
        $source = "access_log_15feb" ;
        @names = ("host","day","month","year","hour","minute",
                        "second","houroff","method","url"
                        ,"status","size", "referer","browser");
        @show = ("when","host","url","status","size", "referer");
        %mnames = (Jan => 0, Feb => 1, Mar => 2, Apr => 3,
                May => 4, Jun => 5, Jul => 6, Aug => 7,
                Sep => 8, Oct => 9, Nov => 10, Dec => 11);
        }