1 file changed, 80 insertions(+), 53 deletions(-)
New commits:
commit a5e636bfff12098b0b73e8efbb44e8097c0c78b5
Author: Mark Reynolds <mreynolds@redhat.com>
Date: Wed May 28 14:38:03 2014 -0400
Ticket 47446 - logconv.pl memory continually grows
Bug Description: Running logconv.pl without any special options
continually consumes more memory.
Fix Description: The main "leak" was a file descriptor array that
was constantly growing with every connection, but
the array served no purpose - so it was removed.
Some other minor changes were made: output formating
improvements, making sure verbose hashes were not
being updated if the verbose optoin was not enabled,
and not calling the CSV report fiunctions if a report
was not being asked for.
https://fedorahosted.org/389/ticket/47446
Reviewed by: nhosoi(Thanks!)
(cherry picked from commit 805386caff210ac07a428b38e99ea7b853bece1f)
(cherry picked from commit 519c58e8082ef093e676eb7a1aaaec03a739ea69)
diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl
index ea33544..edc5fd1 100755
--- a/ldap/admin/src/logconv.pl
+++ b/ldap/admin/src/logconv.pl
@@ -56,7 +56,7 @@ Getopt::Long::Configure ("bundling");
Getopt::Long::Configure ("permute");
if ($#ARGV < 0){;
-&displayUsage;
+ &displayUsage;
}
#######################################
@@ -78,8 +78,8 @@ my $dataLocation = "/tmp";
my $startTLSoid = "1.3.6.1.4.1.1466.20037";
my @statnames=qw(last last_str results srch add mod modrdn moddn cmp del abandon
conns sslconns bind anonbind unbind notesA notesU etime);
-my $s_stats = new_stats_block( );
-my $m_stats = new_stats_block( );
+my $s_stats;
+my $m_stats;
my $verb = "no";
my @excludeIP;
my $xi = 0;
@@ -99,7 +99,6 @@ my %bindReport;
my @vlvconn;
my @vlvop;
my @fds;
-my $fdds = 0;
my $reportBinds = "no";
my $rootDN = "";
my $needCleanup = 0;
@@ -392,34 +391,28 @@ sub statusreport {
##########################################
if ($files[$#files] =~ m/access.rotationinfo/) { $file_count--; }
+$logCount = $file_count;
+
+print "Processing $logCount Access Log(s)...\n";
-print "Processing $file_count Access Log(s)...\n\n";
#print "Filename\t\t\t Total Lines\n";
#print "--------------------------------------------------\n";
my $skipFirstFile = 0;
-if ($file_count > 1 && $files[0] =~ /\/access$/){
- $files[$file_count] = $files[0];
- $file_count++;
- $skipFirstFile = 1;
+if ($logCount > 1 && $files[0] =~ /\/access$/){
+ $files[$logCount] = $files[0];
+ $skipFirstFile = 1;
+ $file_count++;
}
-$logCount = $file_count;
+
my $logline;
my $totalLineCount = 0;
for (my $count=0; $count < $file_count; $count++){
# we moved access to the end of the list, so if its the first file skip it
- if($file_count > 1 && $count == 0 && $skipFirstFile == 1){
- next;
- }
- if($file_count > 1 && $count == 0 && $skipFirstFile == 1){
- next;
- }
- if (-z $files[$count]){
- # access log is empty
- print "Skipping empty access log ($files[$count])...\n";
+ if($logCount > 1 && $count == 0 && $skipFirstFile == 1){
next;
}
$linesProcessed = 0; $lineBlockCount = 0;
@@ -428,6 +421,12 @@ for (my $count=0; $count < $file_count; $count++){
$atime,$mtime,$ctime,$blksize,$blocks) = stat($files[$count]);
print sprintf "[%03d] %-30s\tsize (bytes): %12s\n",$logCount, $files[$count], $cursize;
$logCount--;
+ if ($cursize == 0){
+ # access log is empty
+ print "Skipping empty access log ($files[$count])...\n";
+ next;
+ }
+
my $logCountStr;
if($logCount < 10 ){
# add a zero for formatting purposes
@@ -612,6 +611,12 @@ if($reportBinds eq "yes"){
&displayBindReport();
}
+if($reportStats ne ""){
+ # No need to print the general report, so exit here
+ removeDataFiles();
+ exit (0);
+}
+
#
# Continue with standard report
#
@@ -1366,9 +1371,9 @@ if ($usage =~ /j/i || $verb eq "yes"){
# We're done, clean up the data files
#
removeDataFiles();
-
exit (0);
+
#######################
# #
# Display Usage #
@@ -1524,7 +1529,11 @@ parseLineBind {
$bindReport{$dn}{"failedBind"} = 0;
}
if ($_ =~ /conn= *([0-9A-Z]+)/i) {
- $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 ";
+ if($bindReport{$dn}{"conn"}){
+ $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 ";
+ } else {
+ $bindReport{$dn}{"conn"} = $1;
+ }
}
return;
}
@@ -1645,7 +1654,7 @@ sub parseLineNormal
# Additional performance stats
my ($time, $tzone) = split (' ', $_);
- if (($reportStats or ($verb eq "yes") || ($usage =~ /y/)) && (!defined($last_tm) or ($time ne $last_tm)))
+ if (($reportStats || ($verb eq "yes") || ($usage =~ /y/)) && (!defined($last_tm) or ($time ne $last_tm)))
{
$last_tm = $time;
$time =~ s/\[//;
@@ -1685,12 +1694,16 @@ sub parseLineNormal
if($reportStats){ inc_stats('srch',$s_stats,$m_stats); }
if ($_ =~ / attrs=\"(.*)\"/i){
$anyAttrs++;
- my $attr = $hashes->{attr};
- map { $attr->{$_}++ } split /\s/, $1;
+ if ($usage =~ /r/i || $verb eq "yes"){
+ my $attr = $hashes->{attr};
+ map { $attr->{$_}++ } split /\s/, $1;
+ }
}
if (/ attrs=ALL/){
- my $attr = $hashes->{attr};
- $attr->{"All Attributes"}++;
+ if ($usage =~ /r/i || $verb eq "yes"){
+ my $attr = $hashes->{attr};
+ $attr->{"All Attributes"}++;
+ }
$anyAttrs++;
}
if ($verb eq "yes"){
@@ -1768,16 +1781,20 @@ sub parseLineNormal
$bindCount++;
if($reportStats){ inc_stats('bind',$s_stats,$m_stats); }
if ($1 ne ""){
- $tmpp = $1;
- $tmpp =~ tr/A-Z/a-z/;
- $hashes->{bindlist}->{$tmpp}++;
if($1 eq $rootDN){
$rootDNBindCount++;
}
+ if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){
+ $tmpp = $1;
+ $tmpp =~ tr/A-Z/a-z/;
+ $hashes->{bindlist}->{$tmpp}++;
+ }
} else {
$anonymousBindCount++;
- $hashes->{bindlist}->{"Anonymous Binds"}++;
- inc_stats('anonbind',$s_stats,$m_stats);
+ if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){
+ $hashes->{bindlist}->{"Anonymous Binds"}++;
+ if($reportStats){ inc_stats('anonbind',$s_stats,$m_stats); }
+ }
}
}
if (m/ connection from/){
@@ -1796,15 +1813,20 @@ sub parseLineNormal
if ($simConnection > $maxsimConnection) {
$maxsimConnection = $simConnection;
}
- ($connID) = $_ =~ /conn=(\d*)\s/;
- $openConnection{$connID} = $ip;
- if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) {
- $hashes->{start_time_of_connection}->{$connID} = $gmtime;
+ if ($verb eq "yes" || $usage =~ /p/ || $reportStats){
+ ($connID) = $_ =~ /conn=(\d*)\s/;
+ $openConnection{$connID} = $ip;
+ if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) {
+ $hashes->{start_time_of_connection}->{$connID} = $gmtime;
+ }
}
}
if (m/ SSL client bound as /){$sslClientBindCount++;}
if (m/ SSL failed to map client certificate to LDAP DN/){$sslClientFailedCount++;}
- if (m/ fd=/ && m/slot=/){$fdTaken++}
+ if (m/slot=/ && $_ =~ /fd= *([0-9]+)/i) {
+ $fdTaken++;
+ if ($1 > $highestFdTaken){ $highestFdTaken = $1; }
+ }
if (m/ fd=/ && m/closed/){
($connID) = $_ =~ /conn=(\d*)\s/;
handleConnClose($connID);
@@ -1819,20 +1841,20 @@ sub parseLineNormal
$op = $2;
}
if ($binddn ne ""){
- if($binddn eq $rootDN){$rootDNBindCount++;}
- $tmpp = $binddn;
- $tmpp =~ tr/A-Z/a-z/;
- $hashes->{bindlist}->{$tmpp}++;
- if($usage =~ /f/ || $verb eq "yes"){
+ if($binddn eq $rootDN){ $rootDNBindCount++; }
+ if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){
+ $tmpp = $binddn;
+ $tmpp =~ tr/A-Z/a-z/;
+ $hashes->{bindlist}->{$tmpp}++;
$hashes->{bind_conn_op}->{"$serverRestartCount,$conn,$op"} = $tmpp;
}
} else {
$anonymousBindCount++;
- $hashes->{bindlist}->{"Anonymous Binds"}++;
- if($usage =~ /f/ || $verb eq "yes"){
+ if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){
+ $hashes->{bindlist}->{"Anonymous Binds"}++;
$hashes->{bind_conn_op}->{"$serverRestartCount,$conn,$op"} = "";
}
- inc_stats('anonbind',$s_stats,$m_stats);
+ if($reportStats){ inc_stats('anonbind',$s_stats,$m_stats); }
}
}
if (m/ UNBIND/){
@@ -2109,20 +2131,30 @@ sub parseLineNormal
}
}
if ($_ =~ /err= *([0-9]+)/i){
- $errorCode[$1]++;
+ if ($usage =~ /e/i || $verb eq "yes"){ $errorCode[$1]++; }
if ($1 ne "0"){ $errorCount++;}
else { $successCount++;}
}
- if ($_ =~ /etime= *([0-9.]+)/ ) { $hashes->{etime}->{$1}++; inc_stats_val('etime',$1,$s_stats,$m_stats); }
+ if ($_ =~ /etime= *([0-9.]+)/ ) {
+ my $etime_val = $1;
+ if ($usage =~ /t/i || $verb eq "yes"){ $hashes->{etime}->{$etime_val}++; }
+ if ($reportStats){ inc_stats_val('etime',$etime_val,$s_stats,$m_stats); }
+ }
if ($_ =~ / tag=101 / || $_ =~ / tag=111 / || $_ =~ / tag=100 / || $_ =~ / tag=115 /){
- if ($_ =~ / nentries= *([0-9]+)/i ){ $hashes->{nentries}->{$1}++; }
+ if ($_ =~ / nentries= *([0-9]+)/i ){
+ if ($usage =~ /n/i || $verb eq "yes"){
+ $hashes->{nentries}->{$1}++;
+ }
+ }
}
if (m/objectclass=\*/i || m/objectclass=top/i ){
if (m/ scope=2 /){ $objectclassTopCount++;}
}
if (m/ EXT oid=/){
$extopCount++;
- if ($_ =~ /oid=\" *([0-9\.]+)/i ){ $hashes->{oid}->{$1}++; }
+ if ($_ =~ /oid=\" *([0-9\.]+)/i ){
+ if ($usage =~ /x/i || $verb eq "yes"){$hashes->{oid}->{$1}++; }
+ }
if ($1 && $1 eq $startTLSoid){$startTLSCount++;}
if ($verb eq "yes"){
if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{ext_conn_op}->{"$serverRestartCount,$1,$2"}++;}
@@ -2182,11 +2214,6 @@ sub parseLineNormal
$scopeCount++;
}
}
- if ($_ =~ /fd= *([0-9]+)/i ) {
- $fds[$fdds] = $1;
- if ($fds[$fdds] > $highestFdTaken) {$highestFdTaken = $fds[$fdds];}
- $fdds++;
- }
if ($usage =~ /f/ || $verb eq "yes"){
if (/ err=49 tag=/ && / dn=\"/){
if ($_ =~ /dn=\"(.*)\"/i ){
--
389 commits mailing list
389-commits@lists.fedoraproject.org
https://admin.fedoraproject.org/mailman/listinfo/389-commits
No comments:
Post a Comment