[0001]
[0002]
[0003]
[0004]
[0005]
[0006]
[0007]
[0008]
[0009]
[0010]
[0011]
[0012]
[0013]
[0014]
[0015]
[0016]
[0017]
[0018]
[0019]
[0020]
[0021]
[0022]
[0023]
[0024]
[0025]
[0026]
[0027]
[0028]
[0029]
[0030]
[0031]
[0032]
[0033]
[0034]
[0035]
[0036]
[0037]
[0038]
[0039]
[0040]
[0041]
[0042]
[0043]
[0044]
[0045]
[0046]
[0047]
[0048]
[0049]
[0050]
[0051]
[0052]
[0053]
[0054]
[0055]
[0056]
[0057]
[0058]
[0059]
[0060]
[0061]
[0062]
[0063]
[0064]
[0065]
[0066]
[0067]
[0068]
[0069]
[0070]
[0071]
[0072]
[0073]
[0074]
[0075]
[0076]
[0077]
[0078]
[0079]
[0080]
[0081]
[0082]
[0083]
[0084]
[0085]
[0086]
[0087]
[0088]
[0089]
[0090]
[0091]
[0092]
[0093]
[0094]
[0095]
[0096]
[0097]
[0098]
[0099]
[0100]
[0101]
[0102]
[0103]
[0104]
[0105]
[0106]
[0107]
[0108]
[0109]
[0110]
[0111]
[0112]
[0113]
[0114]
[0115]
[0116]
[0117]
[0118]
[0119]
[0120]
[0121]
[0122]
[0123]
[0124]
[0125]
[0126]
[0127]
[0128]
[0129]
[0130]
[0131]
[0132]
[0133]
[0134]
[0135]
[0136]
[0137]
[0138]
[0139]
[0140]
[0141]
[0142]
[0143]
[0144]
[0145]
[0146]
[0147]
[0148]
[0149]
[0150]
[0151]
[0152]
[0153]
[0154]
[0155]
[0156]
[0157]
[0158]
[0159]
[0160]
[0161]
[0162]
[0163]
[0164]
[0165]
[0166]
[0167]
[0168]
[0169]
[0170]
[0171]
[0172]
[0173]
[0174]
[0175]
[0176]
[0177]
[0178]
[0179]
[0180]
[0181]
[0182]
[0183]
[0184]
[0185]
[0186]
[0187]
[0188]
[0189]
[0190]
[0191]
[0192]
[0193]
[0194]
[0195]
[0196]
[0197]
[0198]
[0199]
[0200]
[0201]
[0202]
[0203]
[0204]
[0205]
[0206]
[0207]
[0208]
[0209]
[0210]
[0211]
[0212]
[0213]
[0214]
[0215]
[0216]
[0217]
[0218]
[0219]
[0220]
[0221]
[0222]
[0223]
[0224]
[0225]
[0226]
[0227]
[0228]
[0229]
[0230]
[0231]
[0232]
[0233]
[0234]
[0235]
[0236]
[0237]
[0238]
[0239]
[0240]
[0241]
[0242]
[0243]
[0244]
[0245]
[0246]
[0247]
[0248]
[0249]
[0250]
[0251]
[0252]
[0253]
[0254]
[0255]
[0256]
[0257]
[0258]
[0259]
[0260]
[0261]
[0262]
[0263]
[0264]
[0265]
[0266]
[0267]
[0268]
[0269]
[0270]
[0271]
[0272]
[0273]
[0274]
[0275]
[0276]
[0277]
[0278]
[0279]
[0280]
[0281]
[0282]
[0283]
[0284]
[0285]
[0286]
[0287]
[0288]
[0289]
[0290]
[0291]
[0292]
[0293]
[0294]
[0295]
[0296]
[0297]
[0298]
[0299]
[0300]
[0301]
[0302]
[0303]
[0304]
[0305]
[0306]
[0307]
[0308]
[0309]
[0310]
[0311]
[0312]
[0313]
[0314]
[0315]
[0316]
[0317]
[0318]
[0319]
[0320]
[0321]
[0322]
[0323]
[0324]
[0325]
[0326]
[0327]
[0328]
[0329]
[0330]
[0331]
[0332]
[0333]
[0334]
[0335]
[0336]
[0337]
[0338]
[0339]
[0340]
[0341]
[0342]
[0343]
[0344]
[0345]
[0346]
[0347]
[0348]
[0349]
[0350]
[0351]
[0352]
[0353]
[0354]
[0355]
[0356]
[0357]
[0358]
[0359]
[0360]
[0361]
[0362]
[0363]
[0364]
[0365]
[0366]
[0367]
[0368]
[0369]
[0370]
[0371]
[0372]
[0373]
[0374]
[0375]
[0376]
[0377]
[0378]
[0379]
[0380]
[0381]
[0382]
[0383]
[0384]
[0385]
[0386]
[0387]
[0388]
[0389]
[0390]
[0391]
[0392]
[0393]
[0394]
[0395]
[0396]
[0397]
[0398]
[0399]
[0400]
[0401]
[0402]
[0403]
[0404]
[0405]
[0406]
[0407]
[0408]
[0409]
[0410]
[0411]
[0412]
[0413]
[0414]
[0415]
[0416]
[0417]
[0418]
[0419]
[0420]
[0421]
[0422]
[0423]
[0424]
[0425]
[0426]
[0427]
[0428]
[0429]
[0430]
[0431]
[0432]
[0433]
[0434]
[0435]
[0436]
[0437]
[0438]
[0439]
[0440]
[0441]
[0442]
[0443]
[0444]
[0445]
[0446]
[0447]
[0448]
[0449]
[0450]
[0451]
[0452]
[0453]
[0454]
[0455]
[0456]
[0457]
[0458]
[0459]
[0460]
[0461]
[0462]
[0463]
[0464]
[0465]
[0466]
[0467]
[0468]
[0469]
[0470]
[0471]
[0472]
[0473]
[0474]
[0475]
[0476]
[0477]
[0478]
[0479]
[0480]
[0481]
[0482]
[0483]
[0484]
[0485]
[0486]
[0487]
[0488]
[0489]
[0490]
[0491]
[0492]
[0493]
[0494]
[0495]
[0496]
[0497]
[0498]
[0499]
[0500]
[0501]
[0502]
[0503]
[0504]
[0505]
[0506]
[0507]
[0508]
[0509]
[0510]
[0511]
[0512]
[0513]
[0514]
[0515]
[0516]
[0517]
[0518]
[0519]
[0520]
[0521]
[0522]
[0523]
[0524]
[0525]
[0526]
[0527]
[0528]
[0529]
[0530]
[0531]
[0532]
[0533]
[0534]
[0535]
[0536]
[0537]
[0538]
[0539]
[0540]
[0541]
[0542]
[0543]
[0544]
[0545]
[0546]
[0547]
[0548]
[0549]
[0550]
[0551]
[0552]
[0553]
[0554]
[0555]
[0556]
[0557]
[0558]
[0559]
[0560]
[0561]
[0562]
[0563]
[0564]
[0565]
[0566]
[0567]
[0568]
[0569]
[0570]
[0571]
[0572]
[0573]
[0574]
[0575]
[0576]
[0577]
[0578]
[0579]
[0580]
[0581]
[0582]
[0583]
[0584]
[0585]
[0586]
[0587]
[0588]
[0589]
[0590]
[0591]
[0592]
[0593]
[0594]
[0595]
[0596]
[0597]
[0598]
[0599]
[0600]
[0601]
[0602]
[0603]
[0604]
[0605]
[0606]
[0607]
[0608]
[0609]
[0610]
[0611]
[0612]
[0613]
[0614]
[0615]
[0616]
[0617]
[0618]
[0619]
[0620]
[0621]
[0622]
[0623]
[0624]
[0625]
[0626]
[0627]
[0628]
[0629]
[0630]
[0631]
[0632]
[0633]
[0634]
[0635]
[0636]
[0637]
[0638]
[0639]
[0640]
[0641]
[0642]
[0643]
[0644]
[0645]
[0646]
[0647]
[0648]
[0649]
[0650]
[0651]
[0652]
[0653]
[0654]
[0655]
[0656]
[0657]
[0658]
[0659]
[0660]
[0661]
[0662]
[0663]
[0664]
[0665]
[0666]
[0667]
[0668]
[0669]
[0670]
[0671]
[0672]
[0673]
[0674]
[0675]
[0676]
[0677]
[0678]
[0679]
[0680]
[0681]
[0682]
[0683]
[0684]
[0685]
[0686]
[0687]
[0688]
[0689]
[0690]
[0691]
[0692]
[0693]
[0694]
[0695]
[0696]
[0697]
[0698]
[0699]
[0700]
[0701]
[0702]
[0703]
[0704]
[0705]
[0706]
[0707]
[0708]
[0709]
[0710]
[0711]
[0712]
[0713]
[0714]
[0715]
[0716]
[0717]
[0718]
[0719]
[0720]
[0721]
[0722]
[0723]
[0724]
[0725]
[0726]
[0727]
[0728]
[0729]
[0730]
[0731]
[0732]
[0733]
[0734]
[0735]
[0736]
[0737]
[0738]
[0739]
[0740]
[0741]
[0742]
[0743]
[0744]
[0745]
[0746]
[0747]
[0748]
[0749]
[0750]
[0751]
[0752]
[0753]
[0754]
[0755]
[0756]
[0757]
[0758]
[0759]
[0760]
[0761]
[0762]
[0763]
[0764]
[0765]
[0766]
[0767]
[0768]
[0769]
[0770]
[0771]
[0772]
[0773]
[0774]
[0775]
[0776]
[0777]
[0778]
[0779]
[0780]
[0781]
[0782]
[0783]
[0784]
[0785]
[0786]
[0787]
[0788]
[0789]
[0790]
[0791]
[0792]
[0793]
[0794]
[0795]
[0796]
[0797]
[0798]
[0799]
[0800]
[0801]
[0802]
[0803]
[0804]
[0805]
[0806]
[0807]
[0808]
[0809]
[0810]
[0811]
[0812]
[0813]
[0814]
[0815]
[0816]
[0817]
[0818]
[0819]
[0820]
[0821]
[0822]
[0823]
[0824]
[0825]
[0826]
[0827]
[0828]
[0829]
[0830]
[0831]
[0832]
[0833]
[0834]
[0835]
[0836]
[0837]
[0838]
[0839]
[0840]
[0841]
[0842]
[0843]
[0844]
[0845]
[0846]
[0847]
[0848]
[0849]
[0850]
[0851]
[0852]
[0853]
[0854]
[0855]
[0856]
[0857]
[0858]
[0859]
[0860]
[0861]
[0862]
[0863]
[0864]
[0865]
[0866]
[0867]
[0868]
[0869]
[0870]
[0871]
[0872]
[0873]
[0874]
[0875]
[0876]
[0877]
[0878]
[0879]
[0880]
[0881]
[0882]
[0883]
[0884]
[0885]
[0886]
[0887]
[0888]
[0889]
[0890]
[0891]
[0892]
[0893]
[0894]
[0895]
[0896]
[0897]
[0898]
[0899]
[0900]
[0901]
[0902]
[0903]
[0904]
[0905]
[0906]
[0907]
[0908]
[0909]
[0910]
[0911]
[0912]
[0913]
[0914]
[0915]
[0916]
[0917]
[0918]
[0919]
[0920]
[0921]
[0922]
[0923]
[0924]
[0925]
[0926]
[0927]
[0928]
[0929]
[0930]
[0931]
[0932]
[0933]
[0934]
[0935]
[0936]
[0937]
[0938]
[0939]
[0940]
[0941]
[0942]
[0943]
[0944]
[0945]
[0946]
[0947]
[0948]
[0949]
[0950]
[0951]
[0952]
[0953]
[0954]
[0955]
[0956]
[0957]
[0958]
/*****************************************************************************/
/*
calogs.c
Consolidate Access LOGS (pronounced the same as the breakfast cereal brand :^)
merges multiple HTTP server common and combined format access logs into a
single log file with records in time-order. Although specifically intended for
per-instance logging (where one log per server process is generated) it can be
used to consolidate any set of access logs. Due to the granularity of HTTP
server entry timestamps (one second) the records are sorted to the one second
but not within the one second.
It uses RMS and the VMS sort-merge routines to provide it's basic consolidation
functionality. An RMS search matches a wildcard log file specification as the
first CLI parameter. Matching files are opened and each record read. The
date/time field is parsed and the textual timestamp converted to a binary time
in seconds (Unix time) which is prepended to the log record and passed to
sort-merge input. When all files have been processed the sort/merge is
performed using the integer time value as an efficient sort key. The sorted
records are then written to the output file specified as the second CLI
parameter (defaults to SYS$OUTPUT) stripping the leading binary time.
To allow some limited filtering of records during the merge the /NOWASD
qualifier excludes WASD server status records (startup, shutdown, timestamp,
etc.), the /NOPROXY qualifier excludes proxy records (producing a log
containing only non-proxy request processing), and the /PROXY qualifier
excludes non-proxy records (producing a log containing only proxy request
processing).
USAGE
-----
$ CALOGS == "$HT_EXE:CALOGS"
$ CALOGS <log-file-spec> [<output-file-name>] [qualifiers]
For example:
$ CALOGS HT_LOGS:*200205*.LOG 2002_MAY.LOG
$ CALOGS /VERBOSE HT_LOGS:
$ CALOGS /NOWASD HT_LOGS:*200206*.LOG_* 2002_JUNE.LOG
$ CALOGS /PROXY /NOWASD HT_LOGS:*2002*.LOG 2002_PROXY.LOG
QUALIFIERS
----------
/DBUG turns on all "if (Debug)" statements
/HELP provide some basic usage information
/NOPROXY only merge records that are not of proxy access
/NOWASD filter out the WASD server status messages
/OUTPUT=<filename> same as second parameter (for habitual users of /OUT= :^)
/PROXY only merge records of proxy access
/QUIET no output except error messages
/SOFTWAREID display the utility version
/VERBOSE provide file-by-file statistics
/VERSION display the utility version
BUILD DETAILS
-------------
See BUILD_CALOGS.COM procedure.
COPYRIGHT
---------
Copyright (C) 2002-2021 Mark G.Daniel
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
VERSION HISTORY (update SOFTWAREVN as well!)
---------------
23-DEC-2003 MGD v1.0.1, minor conditional mods to support IA64
28-MAY-2002 MGD v1.0.0, initial development
*/
/*****************************************************************************/
#define SOFTWAREVN "1.0.1"
#define SOFTWARENM "CALOGS"
#ifdef __ALPHA
# define SOFTWAREID SOFTWARENM " AXP-" SOFTWAREVN
#endif
#ifdef __ia64
# define SOFTWAREID SOFTWARENM " IA64-" SOFTWAREVN
#endif
#ifdef __VAX
# error VAX no longer implemented
#endif
#ifdef __x86_64
# define SOFTWAREID SOFTWARENM " X86-" SOFTWAREVN
#endif
/* standard C header files */
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
/* VMS-related header files */
#include <descrip.h>
#include <lib$routines.h>
#include <rms.h>
#include <sordef.h>
#include <sor$routines.h>
#include <ssdef.h>
#include <stsdef.h>
#include <starlet.h>
#define BOOL int
#define true 1
#define false 0
#define VMSok(x) ((x) & STS$M_SUCCESS)
#define VMSnok(x) (!((x) & STS$M_SUCCESS))
/* maximum log line/record length */
#define MAX_LINE_LENGTH 2048
char CopyrightInfo [] =
"Copyright (C) 2002-2021 Mark G.Daniel.\n\
This software comes with ABSOLUTELY NO WARRANTY.\n\
This is free software, and you are welcome to redistribute it\n\
under the conditions of the GNU GENERAL PUBLIC LICENSE, version 2.";
char Utility [] = "CALOGS";
BOOL Debug,
DoNoWasd,
DoNoProxy,
DoProxyOnly,
DoQuiet,
DoVerbose;
int DiscardRecordCount,
FileCount,
InputRecordCount,
NonProxyRecordCount,
OutputRecordCount,
ProxyRecordCount,
ReadRecordCount,
WasdRecordCount;
char LogFileSpec [256],
MergedFileName [256];
typedef struct
{
unsigned short type;
unsigned short order;
unsigned short offset;
unsigned short len;
} KEY_INFO;
struct
{
unsigned short num;
KEY_INFO key [1];
} SortKeys;
globalvalue int SOR$M_STABLE;
/* required function prototypes */
int GetParameters ();
int ShowHelp ();
int ReadLogFiles ();
int ProcessLogFile (char*);
int WriteMergedLog ();
int strsame (char*, char*, int);
char* SysGetMsg (int);
/*****************************************************************************/
/*
*/
main ()
{
int status,
SortLrl,
SortOptions;
/*********/
/* begin */
/*********/
if (getenv ("CALOGS$DBUG")) Debug = true;
GetParameters ();
if (DoVerbose)
fprintf (stdout, "%%%s-I-VERBOSE, version %s\n", Utility, SOFTWAREID);
/* time (Unix seconds) */
SortKeys.key[0].type = DSC$K_DTYPE_LU;
SortKeys.key[0].order = 0;
SortKeys.key[0].offset = 0;
SortKeys.key[0].len = 4;
SortKeys.num = 1;
SortOptions = SOR$M_STABLE;
SortLrl = MAX_LINE_LENGTH + 4;
if (VMSnok (status =
sor$begin_sort (&SortKeys, &SortLrl, &SortOptions,
0, 0, 0, 0, 0, 0)))
{
fprintf (stdout, "%%%s-E-SORT, begin\n-%s\n",
Utility, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
if (DoVerbose) fprintf (stdout, "READING ...\n");
ReadLogFiles ();
if (DoVerbose) fprintf (stdout, "MERGING ...\n");
if (VMSnok (status = sor$sort_merge (0)))
{
fprintf (stdout, "%%%s-E-SORT, merge\n-%s\n",
Utility, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
if (InputRecordCount)
{
if (DoVerbose) fprintf (stdout, "WRITING ...\n");
if (!MergedFileName[0]) strcpy (MergedFileName, "SYS$OUTPUT:");
WriteMergedLog ();
}
if (VMSnok (status = sor$end_sort (0)))
{
fprintf (stdout, "%%%s-E-SORT, end\n-%s\n",
Utility, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
if (!DoQuiet || DoVerbose)
{
fprintf (stdout,
"%%%s-%s-STATISTICS, for log consolidation\n\
Log Files: %d\n\
Records Read: %d\n\
Discarded: %d\n",
Utility,
InputRecordCount == OutputRecordCount ? "I" : "W",
FileCount, ReadRecordCount, DiscardRecordCount);
if (DoNoProxy || DoProxyOnly)
fprintf (stdout,
" Proxy: %d\n\
Non-Proxy: %d\n",
ProxyRecordCount, NonProxyRecordCount);
if (DoNoWasd)
fprintf (stdout,
" WASD: %d\n",
WasdRecordCount);
fprintf (stdout,
" Merged: %d\n\
Output: %d\n",
InputRecordCount, OutputRecordCount);
if (InputRecordCount != OutputRecordCount)
fprintf (stdout, "-%s-W-COUNT, input and output record count?\n",
Utility);
}
if (InputRecordCount != OutputRecordCount) exit (SS$_BUGCHECK);
exit (SS$_NORMAL);
}
/****************************************************************************/
/*
Search against the command line file specification, passing each file found to
be processed.
*/
ReadLogFiles ()
{
int status,
FileNameLength,
Length;
char FileName [256],
ExpFileName [256];
struct FAB SearchFab;
struct NAM SearchNam;
/*********/
/* begin */
/*********/
if (Debug) fprintf (stdout, "ReadLogFiles()\n");
/* initialize the file access block */
SearchFab = cc$rms_fab;
SearchFab.fab$l_dna = "*.LOG*;";
SearchFab.fab$b_dns = 7;
SearchFab.fab$l_fna = LogFileSpec;
SearchFab.fab$b_fns = strlen(LogFileSpec);
SearchFab.fab$l_nam = &SearchNam;
SearchNam = cc$rms_nam;
SearchNam.nam$l_esa = ExpFileName;
SearchNam.nam$b_ess = sizeof(ExpFileName)-1;
SearchNam.nam$l_rsa = FileName;
SearchNam.nam$b_rss = sizeof(FileName)-1;
if (VMSnok (status = sys$parse (&SearchFab, 0, 0)))
{
fprintf (stdout, "%%%s-E-PARSE, %s\n-%s\n",
Utility, LogFileSpec, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
while (VMSok (status = sys$search (&SearchFab, 0, 0)))
{
FileCount++;
*SearchNam.nam$l_ver = '\0';
ProcessLogFile (FileName);
*SearchNam.nam$l_ver = ';';
}
if (VMSnok (status) && status != RMS$_NMF)
{
SearchNam.nam$l_ver[SearchNam.nam$b_ver] = '\0';
fprintf (stdout, "%%%s-E-SEARCH, %s\n-%s\n",
Utility, FileName, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
}
/****************************************************************************/
/*
Open the log file, read and process each record from it, close the file!
Parse the date/time components from each record, converting them into a binary
integer value (Unix time in seconds). Prepend this this to the textual
component of the record passing this to the sort-merge utility routines. To
reduce unnecessary copying of record contents the buffer is organized so the
first four bytes are left available for prepending the integer time.
The common and combined formats ...
'client rfc891 authuser [date/time] "request" status bytes'
'client rfc891 authuser [date/time] "request" status bytes referer agent'
*/
int ProcessLogFile (char *FileName)
{
static char *MonthAbbrev [] = { NULL,
"Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
static $DESCRIPTOR (RecordDsc, "");
int status,
day, month, year, hour, min, sec,
FileDiscardRecordCount,
FileInputRecordCount,
FileNonProxyRecordCount,
FileProxyRecordCount,
FileRecordCount,
FileWasdRecordCount,
UnixTime;
char *cptr, *mptr, *sptr;
/* plus four allows for the binary timestamp */
char Line [MAX_LINE_LENGTH+4];
struct tm utm;
struct FAB FileFab;
struct RAB FileRab;
/*********/
/* begin */
/*********/
if (Debug) fprintf (stdout, "ProcessLogFile()\n");
if (DoVerbose) fprintf (stdout, "%s\n", FileName);
FileFab = cc$rms_fab;
FileFab.fab$b_fac = FAB$M_GET;
FileFab.fab$l_fna = FileName;
FileFab.fab$b_fns = strlen(FileName);
FileFab.fab$b_shr = FAB$M_SHRGET | FAB$M_SHRPUT | FAB$M_SHRUPD;
if (VMSnok (status = sys$open (&FileFab, 0, 0)))
{
if (status == RMS$_FLK)
{
/* convert it into a warning */
status &= 0xfffffff8;
fprintf (stdout, "%%%s-W-OPEN, %s\n-%s\n",
Utility, FileName, SysGetMsg(status)+1);
return (status);
}
fprintf (stdout, "%%%s-E-OPEN, %s\n-%s\n",
Utility, FileName, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
FileRab = cc$rms_rab;
FileRab.rab$l_fab = &FileFab;
/* 2 buffers and read ahead performance option */
FileRab.rab$b_mbf = 2;
FileRab.rab$l_rop = RAB$M_RAH;
/* plus four, minus five allows for the binary timestamp */
FileRab.rab$l_ubf = (char*)Line + 4;
FileRab.rab$w_usz = sizeof(Line) - 5;
if (VMSnok (status = sys$connect (&FileRab, 0, 0)))
{
fprintf (stdout, "%%%s-E-CONNECT, %s\n-%s\n",
Utility, FileName, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
FileDiscardRecordCount = FileInputRecordCount =
FileNonProxyRecordCount = FileProxyRecordCount =
FileRecordCount = FileWasdRecordCount = 0;
while (VMSok (status = sys$get (&FileRab, 0, 0)))
{
ReadRecordCount++;
FileRecordCount++;
cptr = Line + 4;
cptr[FileRab.rab$w_rsz] = '\0';
if (DoNoProxy || DoProxyOnly || DoNoWasd)
{
/* skip over client and remote ident fields */
while (*cptr && !isspace(*cptr)) cptr++;
while (*cptr && isspace(*cptr)) cptr++;
while (*cptr && !isspace(*cptr)) cptr++;
while (*cptr && isspace(*cptr)) cptr++;
sptr = cptr;
}
if (DoNoProxy || DoProxyOnly)
{
/* skip straight on to the request field */
while (*cptr && *cptr != '\"') cptr++;
if (*cptr) *cptr++;
/* skip over method */
mptr = cptr;
while (*cptr && !isspace(*cptr)) cptr++;
if (*cptr) *cptr++;
if (*cptr == '/')
{
NonProxyRecordCount++;
FileNonProxyRecordCount++;
if (DoProxyOnly) continue;
}
else
if (*mptr == 'C' && !memcmp (mptr, "CONNECT", 7))
{
ProxyRecordCount++;
FileProxyRecordCount++;
if (DoNoProxy) continue;
}
else
{
/* look for what passes for a URL scheme */
while (*cptr && *cptr != ':' && *cptr != '/' && *cptr != '\"')
cptr++;
if (cptr[0] == ':' && cptr[1] == '/' && cptr[2] == '/')
{
ProxyRecordCount++;
FileProxyRecordCount++;
if (DoNoProxy) continue;
}
}
/* go back to the 'authorized user' field */
cptr = sptr;
}
if (DoNoWasd)
{
if (!memcmp (cptr, "HTTPd ", 6))
{
/* authorized user matches, quite possible, look at request */
cptr += 6;
/* skip to the start of the request field */
while (*cptr && *cptr != '\"') cptr++;
if (*cptr == '\"')
{
/* looking for:
pre-8.2 '"POST <node>::HTTP<char>:<port>-<whatever>"'
post-8.2 '"POST /<node>::HTTP<char>:<port>-<whatever>"'
*/
cptr++;
if (!memcmp (cptr, "POST ", 5))
{
/* looking more likely */
cptr += 5;
while (*cptr && *cptr != ':' && !isspace(*cptr)) cptr++;
if (*(unsigned short*)cptr == '::')
{
/* more and more likely :^) */
cptr += 2;
while (*cptr && *cptr != ':' && !isspace(*cptr)) cptr++;
if (*cptr == ':')
{
/* almost certain */
cptr++;
if (isdigit(*cptr))
{
while (*cptr && isdigit(*cptr)) cptr++;
if (*cptr == '-')
{
/* ok, I'm convinced! */
WasdRecordCount++;
FileWasdRecordCount++;
continue;
}
}
}
}
}
}
}
/* go back to the 'authorized user' field */
cptr = sptr;
}
/* straight to the date field */
while (*cptr && *cptr != '[') cptr++;
if (*cptr != '[')
{
/* doesn't look like the start of a date field to me */
DiscardRecordCount++;
FileDiscardRecordCount++;
continue;
}
cptr++;
/* date/time */
year = month = day = hour = min = sec = 0;
day = atoi(cptr);
while (*cptr && *cptr != '/') cptr++;
if (*cptr) cptr++;
for (month = 1; month <= 12; month++)
if (strsame (MonthAbbrev[month], cptr, 3)) break;
while (*cptr && *cptr != '/') cptr++;
if (*cptr) cptr++;
year = atoi(cptr);
while (*cptr && *cptr != ':') cptr++;
if (*cptr) cptr++;
hour = atoi(cptr);
while (*cptr && *cptr != ':') cptr++;
if (*cptr) cptr++;
min = atoi(cptr);
while (*cptr && *cptr != ':') cptr++;
if (*cptr) cptr++;
sec = atoi(cptr);
memset (&utm, 0, sizeof(utm));
utm.tm_sec = sec;
utm.tm_min = min;
utm.tm_hour = hour;
utm.tm_mday = day;
utm.tm_mon = month - 1;
utm.tm_year = year - 1900;
UnixTime = mktime (&utm);
if (Debug)
fprintf (stdout, "%d %d %d %d %d %d = %d\n",
year, month, day, hour, min, sec, UnixTime);
if (UnixTime == -1)
{
DiscardRecordCount++;
FileDiscardRecordCount++;
continue;
}
/* prepend the binary, longword time to the original buffer */
*(int*)Line = UnixTime;
/* plus four allows for the binary timestamp */
RecordDsc.dsc$a_pointer = Line;
RecordDsc.dsc$w_length = FileRab.rab$w_rsz + 4;
if (VMSnok (status = sor$release_rec (&RecordDsc, 0)))
{
fprintf (stdout, "%%%s-E-SORT, release record\n-%s\n",
Utility, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
InputRecordCount++;
FileInputRecordCount++;
}
if (status != RMS$_EOF)
{
fprintf (stdout, "%%%s-E-GET, %s\n-%s\n",
Utility, FileName, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
sys$close (&FileFab, 0, 0);
if (DoVerbose)
{
fprintf (stdout, "records:%d discarded:%d",
FileRecordCount, FileDiscardRecordCount);
if (DoNoWasd) fprintf (stdout, " WASD:%d", FileWasdRecordCount);
if (DoNoProxy || DoProxyOnly)
fprintf (stdout, " proxy:%d non-proxy:%d",
FileProxyRecordCount, FileNonProxyRecordCount);
fprintf (stdout, " input:%d\n", FileInputRecordCount);
}
return (SS$_NORMAL);
}
/****************************************************************************/
/*
Open an output file for write. Retrieve each record from the sort-merge
utility routines. Strip the leading binary time writing the textual component
of the record to the output file.
*/
int WriteMergedLog ()
{
int status;
unsigned short ShortLength;
char ExpFileName [256],
Line [MAX_LINE_LENGTH+4];
struct FAB FileFab;
struct NAM FileNam;
struct RAB FileRab;
$DESCRIPTOR (RecordDsc, Line);
/*********/
/* begin */
/*********/
if (Debug) fprintf (stdout, "WriteMergedLog()\n");
FileFab = cc$rms_fab;
FileFab.fab$l_dna = ".LOG";
FileFab.fab$b_dns = 4;
FileFab.fab$l_fop = FAB$M_DFW | FAB$M_SQO | FAB$M_TEF;
FileFab.fab$l_fna = MergedFileName;
FileFab.fab$b_fns = strlen(MergedFileName);
FileFab.fab$b_fac = FAB$M_PUT;
FileFab.fab$l_nam = &FileNam;
FileFab.fab$b_rfm = FAB$C_STMLF;
FileFab.fab$b_org = FAB$C_SEQ;
FileFab.fab$b_rat = FAB$M_CR;
FileNam = cc$rms_nam;
FileNam.nam$l_esa = ExpFileName;
FileNam.nam$b_ess = sizeof(ExpFileName)-1;
if (VMSnok (status = sys$create (&FileFab, 0, 0)))
{
fprintf (stdout, "%%%s-E-CREATE, %s\n-%s\n",
Utility, MergedFileName, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
FileRab = cc$rms_rab;
FileRab.rab$l_fab = &FileFab;
FileRab.rab$b_mbc = 127;
FileRab.rab$b_mbf = 2;
FileRab.rab$l_rop = RAB$M_WBH;
if (VMSnok (status = sys$connect (&FileRab, 0, 0)))
{
fprintf (stdout, "%%%s-E-CONNECT, %s\n-%s\n",
Utility, MergedFileName, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
*FileNam.nam$l_ver = '\0';
for (;;)
{
if (VMSnok (status = sor$return_rec (&RecordDsc, &ShortLength, 0)))
{
if (status == SS$_ENDOFFILE) break;
fprintf (stdout, "%%%s-E-SORT, return record\n-%s\n",
Utility, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
OutputRecordCount++;
FileRab.rab$l_rbf = Line + 4;
FileRab.rab$w_rsz = ShortLength - 4;
if (VMSnok (status = sys$put (&FileRab, 0, 0)))
{
fprintf (stdout, "%%%s-E-PUT, %s\n-%s\n",
Utility, ExpFileName, SysGetMsg(status)+1);
exit (status | STS$M_INHIB_MSG);
}
}
sys$close (&FileFab, 0, 0);
if (DoVerbose)
fprintf (stdout, "%s\nrecords: %d\n", ExpFileName, OutputRecordCount);
return (SS$_NORMAL);
}
/*****************************************************************************/
/*
Get "command-line" parameters, whether from the command-line or from a
configuration symbol or logical containing the equivalent.
*/
GetParameters ()
{
static char CommandLine [256];
static unsigned long Flags = 0;
int status;
unsigned short Length;
char ch;
char *aptr, *cptr, *clptr, *sptr;
$DESCRIPTOR (CommandLineDsc, CommandLine);
/*********/
/* begin */
/*********/
if (Debug) fprintf (stdout, "GetParameters()\n");
if ((clptr = getenv ("CALOGS$PARAM")) == NULL)
{
/* get the entire command line following the verb */
if (VMSnok (status =
lib$get_foreign (&CommandLineDsc, 0, &Length, &Flags)))
exit (status);
(clptr = CommandLine)[Length] = '\0';
}
aptr = NULL;
ch = *clptr;
for (;;)
{
if (aptr != NULL && *aptr == '/') *aptr = '\0';
if (!ch) break;
*clptr = ch;
if (Debug) fprintf (stdout, "clptr |%s|\n", clptr);
while (*clptr && isspace(*clptr)) *clptr++ = '\0';
aptr = clptr;
if (*clptr == '/') clptr++;
while (*clptr && !isspace (*clptr) && *clptr != '/')
{
if (*clptr != '\"')
{
clptr++;
continue;
}
cptr = clptr;
clptr++;
while (*clptr)
{
if (*clptr == '\"')
if (*(clptr+1) == '\"')
clptr++;
else
break;
*cptr++ = *clptr++;
}
*cptr = '\0';
if (*clptr) clptr++;
}
ch = *clptr;
if (*clptr) *clptr = '\0';
if (Debug) fprintf (stdout, "aptr |%s|\n", aptr);
if (!*aptr) continue;
if (strsame (aptr, "/DBUG", -1))
{
Debug = true;
continue;
}
if (strsame (aptr, "/HELP", -1))
{
ShowHelp ();
exit (SS$_NORMAL);
}
if (strsame (aptr, "/PROXY", 5))
{
DoProxyOnly = true;
continue;
}
if (strsame (aptr, "/NOPROXY", 5))
{
DoNoProxy = true;
continue;
}
if (strsame (aptr, "/NOWASD", 5))
{
DoNoWasd = true;
continue;
}
if (strsame (aptr, "/OUTPUT=", 4))
{
while (*aptr && *aptr != '=') aptr++;
if (*aptr) aptr++;
strcpy (MergedFileName, aptr);
continue;
}
if (strsame (aptr, "/QUIET", 4))
{
DoQuiet = true;
continue;
}
if (strsame (aptr, "/SOFTWAREID", 4) ||
strsame (aptr, "/VERSION", 5))
{
fprintf (stdout, "%%%s-I-SOFTWAREID, %s\n%s\n",
Utility, SOFTWAREID, CopyrightInfo);
exit (SS$_NORMAL);
}
if (strsame (aptr, "/VERBOSE", 5))
{
DoVerbose = true;
continue;
}
if (*aptr == '/')
{
fprintf (stdout, "%%%s-E-IVQUAL, unrecognized qualifier\n \\%s\\\n",
Utility, aptr+1);
exit (STS$K_ERROR | STS$M_INHIB_MSG);
}
if (!LogFileSpec[0])
{
sptr = LogFileSpec;
for (cptr = aptr; *cptr; *sptr++ = toupper(*cptr++));
*sptr = '\0';
continue;
}
if (!MergedFileName[0])
{
sptr = MergedFileName;
for (cptr = aptr; *cptr; *sptr++ = toupper(*cptr++));
*sptr = '\0';
continue;
}
fprintf (stdout, "%%%s-E-MAXPARM, too many parameters\n \\%s\\\n",
Utility, aptr);
exit (STS$K_ERROR | STS$M_INHIB_MSG);
}
}
/****************************************************************************/
/*
Does a case-insensitive, character-by-character string compare and returns
true if two strings are the same, or false if not. If a maximum number of
characters are specified only those will be compared, if the entire strings
should be compared then specify the number of characters as 0.
*/
BOOL strsame
(
char *sptr1,
char *sptr2,
int count
)
{
while (*sptr1 && *sptr2)
{
if (toupper (*sptr1++) != toupper (*sptr2++)) return (false);
if (count)
if (!--count) return (true);
}
if (*sptr1 || *sptr2)
return (false);
else
return (true);
}
/****************************************************************************/
/*
Return a pointer to the textual meaning of the specific VMS status value.
*/
char* SysGetMsg (int StatusValue)
{
static char Message [256];
static $DESCRIPTOR (MessageDsc, Message);
int status;
short int Length;
/*********/
/* begin */
/*********/
if (Debug) fprintf (stdout, "SysGetMsg() %%X%08.08X\n", StatusValue);
status = sys$getmsg (StatusValue, &Length, &MessageDsc, 0, 0);
if (VMSnok (status)) exit (status);
Message[Length] = '\0';
return (Message);
}
/*****************************************************************************/
/*
*/
ShowHelp ()
{
fprintf (stdout,
"Usage for Consolidate Access LOGS (%s)\n\
\n\
Consolidate Access LOGs merges multiple HTTP server common and combined format\n\
access logs into a single log file with records in time-order. Due to the\n\
granularity of HTTP server entry timestamps (one second) the records are sorted\n\
to the one second but not within the one second. The /NOPROXY, /PROXY and\n\
/NOWASD qualifiers allow the particular class of record to be excluded from the\n\
merged file.\n\
\n\
$ CALOGS <log-file-spec> [<output-file-name>] [<qualifiers>]\n\
\n\
/HELP /NOPROXY /NOWASD /OUTPUT=<filename> /PROXY /QUIET /VERBOSE /VERSION\n\
\n\
Usage examples:\n\
\n\
$ CALOGS == \"$dir:CALOGS\"\n\
$ CALOGS HT_LOGS:*200205*.LOG 2002_MAY.LOG\n\
$ CALOGS /VERBOSE HT_LOGS:\n\
$ CALOGS /NOWASD HT_LOGS:*200206*.LOG_* 2002_JUNE.LOG\n\
$ CALOGS /PROXY /NOWASD HT_LOGS:*2002*.LOG 2002_PROXY.LOG\n\
\n",
SOFTWAREID);
}
/*****************************************************************************/