[hcoop/scripts.git] / quotacheck

#!/usr/bin/perl

# docelic, Sat Dec 22 18:35:37 EST 2007

# Script warns people when they're approaching quota. The purpose
# is not to 'behave' users but to give them a notice they should
# fill quota increase request. (We want to prevent people from 
# ever actually running into quota limits, especially for Mail
# and Database volumes. We use quotas to keep the shop clean, not
# to impose restrictions).

# Script retrieves volume info from the afs server, from there it
# reads mail., user. and db. volumes into %vol, then sees which
# volumes have > $threshold usage, relative to volume quota.
#
# For volumes above threshold, it extracts the owner name from
# volume name (like, mail.USERNAME) and saves info to %warn.
#
# vos listvol is parsed into structure like this:
#
# $vol{user.docelic}{diskused} = 36718 ( All fields from the 
#   output of vos listvol are loaded into the hash. list of
#   fields is at the bottom of this script )
#
# people who are over quota are in %warn:
#
# $warn{docelic}{user} = 0.9
# $warn{docelic}{mail} = 0.74
#
# "Offenders" are saved to $state{$user} = [ run_number, exp_inc ]
# and removed when they resolve their quota situation. For this
# to work (and emails to get sent), pass --auto (most certain when
# running from cron).
#
#    run_number is the run (tick) when they were first spotted to
#    be over safe value, and exp_inc is the interval at which they
#    need to be reminded. (We use 1, 2, 4, 8, 16...). Each 'tick' 
#    and number usually refers to 1 day (if ran from cron daily)

#    In the current implimentation, folks will be reminded on
#    1st, 3rd, 7th, 15th and 31st day after their quota overrun.
#    Then they will be given 2 months of no messages, and then 
#    after 3 months all together, the "warning cycle" will start again.

# - If user is over 0.96 on mail or db volume, we will add 2% to his
#    quota one-time to alleviate immediate problem.

use warnings;
use strict;
use Storable qw/nstore retrieve/;
use Fatal qw/open write read close/;
use Data::Dumper qw/Dumper/;

# Warn threshold
our %threshold = ( user => 0.8, mail => 0.6, db => 0.4 );
# AFS servers with volumes
our @servers = qw/gibran/;
# Which volumes we want?
# (Your volumes need to be in format (TYPE).USERNAME, otherwise
# you will have to modify below where we take .USERNAME to mean
# person's USERNAME). Changing this will also trigger changes
# in template email (see DATA below).
our $volume_pattern = '(mail|user)\.[a-zA-Z0-9]+'; # implicit ^...$
# Cache file
our $statefile = '/var/cache/hcoop/quotacheck';
# Hash structure containing load of $statefile
our %state;
# Update cache & send out emails (--auto), or just run terminal report
our $auto = grep {/^--auto$/} @ARGV;
# Dump statefile in readable format to terminal and exit (--dump)
our $dump = grep {/^--dump$/} @ARGV;
# Do not send emails:
our $nomail = grep {/^--nomail$/} @ARGV;
# E-mail template that is evaled before sending
our @template = <DATA>;


# WARMUP

# If statefile is there, eventually load it
if ( -e $statefile ) {
	if ( $auto or $dump) { %state = %{ retrieve($statefile) } }
}

# Only dump wanted?
if (  $dump ) { print Dumper \%state; exit 0 }

# Increment tick
$state{_tick}++;


# BEGIN WOROK

# Get volume lists from all servers
my @list;
for (@servers) { push @list, `vos listvol $_ -format` }
chomp for @list;

my $current; # name of volume being parsed
my %vol; # will contain hash result of parsing
my %warn; # list of people and data for > $threshold folks
my %nowarn; # same data as in %warn, but for people NOT over threshold

#
# Copy vol from @list (vos listvol output) into %vol hash.
# We read in all values; all values from vol listvol -format
# output for a volume are read in. Such as
#  $vol{user.docelic}{type} = 'RW'
#
for (@list) {
	my ($parm, $value) = split(/\s+/, $_, 2);
	defined $parm and defined $value or next;

	if ( $parm eq 'name' ) {
		if ( $value =~ /^$volume_pattern$/ ) {
			$current = $value
		} else {
			$current = ''
		}
	} elsif ($current) {
		$vol{$current}{$parm} = $value
	}
}

#
# Copy > $threshold people to %warn hash like this:
#   $warn{docelic}{mail} = 0.9
#
while (my($k,$v) = each %vol) {
	next unless $$v{maxquota};
	my ($type,$user) = split(/\./, $k);

	my $perc = sprintf('%.3f', $$v{diskused} / $$v{maxquota});

	if ( $perc > $threshold{$type} ) {
		$warn{$user}{$type} = $perc;
	}

	# A bit of duplication with %warn, but wth..
	$nowarn{$user}{$type} = $perc;
}

#
# Reset counter for people who solved their quota thing somehow
#
my @tmp = keys %state;
for my $person ( @tmp ) {
	next if $person eq '_tick';
	unless ( $warn{$person} ) {
		print "User $person resolved quota problem\n";
		delete $state{$person}
	}
}

#
# Go over %warn, print info to terminal, update %state
# when needed to create new entry for new quota detections.
#
while (my($k,$v) = each %warn) {
	print "$k ";
	my @line;
	while (my($k2,$v2) = each %$v) {
		push @line, "$k2=$v2";
	}
	print "@line";

	# Register person in %state
	if ( ! $state{$k} ) {
		$state{$k} = [ $state{_tick}, 2 ];
	}

	print "\n";
}

#
# See who's entitled to receiving an email at this point;
# that is, go over %state, send emails to people who need
# to get it in this turn, and exponentially prolong the time
# till next message from us.
# (They receive message on 1st, 3rd, 7th, 15th, 31st...)
#
while (my($user,$state) = each %state ) {
	next if $user eq '_tick';
	#print " $$state[0] + $$state[1] == $state{_tick} + 2\n";

	if ( $$state[0] + $$state[1] == $state{_tick} + 2 ) {

		if  ($auto and !$nomail) {
			print "Notifying $user, next notice in $$state[1] days.\n";
			email_notify($user);
		}

		$$state[1] *= 2;

		# Following code would re-start the notifications we
		# send to people after 120 days of not clearing
		# their issue, but let's keep it disabled for
		# now.. They get a notice on 127th day, and then
		# on 255th etc., we don't reset the sequence.

		#$$state[1] >= 120 and do {
		#	$$state[0] = $state{_tick};
		#	$$state[1] = 2;
		#};
	}
}


# Write cache
if ( $auto ) {
	# Already done above, with also printing a nice
	# message
	#for ( keys %warn ) {
	#	delete $nowarn{$_}
	#}
	#for ( keys %nowarn) {
	#	delete $state{$_};
	#}

	nstore \%state, $statefile;
}

DONE:
exit 0;
##############################################
# Helpers below


sub email_notify {
	my $user = shift;

	my @msg = @template;
	for ( @msg ) {
		$_ =~ s/\$vol{(\w+)}{(\w+)}/$vol{"$1.$user"}{$2}/mge;
		$_ =~ s/\$threshold{(\w+)}/$threshold{$1}/mge;
		$_ =~ s/\$warn{(\w+)}/$nowarn{$user}{$1}/mge;
		$_ =~ s/\$CRIT{(\w+)}/$warn{$user}{$1} ? " (APPROACHING QUOTA)" : ""/mge;
	}
	#print join('', @msg);
	open(MAIL, "| mail -s 'Approaching quota limit at HCoop' -a 'From: admins\@hcoop.net' -b 'admins\@hcoop.net' '$user\@hcoop.net'");
	{ no warnings; print main::MAIL join('', @msg); }
	close MAIL;
}


# BEGIN_OF_ENTRY
# name            user.docelic
# id              536872079
# serv            69.90.123.67    deleuze.hcoop.net
# part            /vicepa
# status          OK
# backupID        536872081
# parentID        536872079
# cloneID         536955593
# inUse           Y
# needsSalvaged   N
# destroyMe       N
# type            RW
# creationDate    1195841197      Fri Nov 23 13:06:37 2007
# accessDate      0               Wed Dec 31 19:00:00 1969
# updateDate      1198364408      Sat Dec 22 18:00:08 2007
# backupDate      1198322931      Sat Dec 22 06:28:51 2007
# copyDate        1195843557      Fri Nov 23 13:45:57 2007
# flags           0       (Optional)
# diskused        36718
# maxquota        400000
# minquota        0       (Optional)
# filecount       6338
# dayUse          20097
# weekUse         172438  (Optional)
# spare2          0       (Optional)
# spare3          0       (Optional)
# END_OF_ENTRY

__DATA__
Hello,

Some of your data volumes at HCoop are approaching quota
limit.

We use quotas to "keep the shop clean", but we regularly
grant quota increase requests to all members who submit them.

We wouldn't want you hit the quota limit, hence the early
notice. This is especially true for the mail and database
volumes where the data can accumulate "by itself", and where
reaching quota limits may be particularly inconvenient.
If you plan to store more data, it might be a good
time to submit the quota change request through our portal[1].

Here are your volume statistics:

Home:
    size=$vol{user}{diskused} KB
    quota=$vol{user}{maxquota} KB
    threshold=$threshold{user}
    use=$warn{user} $CRIT{user}

Mail:
    size=$vol{mail}{diskused} KB
    quota=$vol{mail}{maxquota} KB
    threshold=$threshold{mail}
    use=$warn{mail} $CRIT{mail}

Please submit your disk quota increase request in a timely
manner at:

  [1] https://members.hcoop.net/portal/quota


These automatic notices will be sent to you at exponential
intervals; that means in 2 days, 4 days, 8 days, 16 days and
so on. The method ensures that, if you retain status-quo
(don't submit a quota increase and don't reduce your data size),
you get to receive the reminders less and less often.
When and if the usage percentage of all your volumes drops below
the corresponding quota thresholds, the notifications will stop
and reset.

Thank you,
HCoop
Commit	Line	Data
63ecf196	1	#!/usr/bin/perl
	2
	3	# docelic, Sat Dec 22 18:35:37 EST 2007
	4
	5	# Script warns people when they're approaching quota. The purpose
	6	# is not to 'behave' users but to give them a notice they should
	7	# fill quota increase request. (We want to prevent people from
	8	# ever actually running into quota limits, especially for Mail
	9	# and Database volumes. We use quotas to keep the shop clean, not
	10	# to impose restrictions).
	11
0cf246e4 CE	12	# Script retrieves volume info from the afs server, from there it
	13	# reads mail., user. and db. volumes into %vol, then sees which
	14	# volumes have > $threshold usage, relative to volume quota.
63ecf196	15	#
	16	# For volumes above threshold, it extracts the owner name from
	17	# volume name (like, mail.USERNAME) and saves info to %warn.
	18	#
	19	# vos listvol is parsed into structure like this:
	20	#
	21	# $vol{user.docelic}{diskused} = 36718 ( All fields from the
	22	# output of vos listvol are loaded into the hash. list of
	23	# fields is at the bottom of this script )
	24	#
	25	# people who are over quota are in %warn:
	26	#
	27	# $warn{docelic}{user} = 0.9
	28	# $warn{docelic}{mail} = 0.74
	29	#
	30	# "Offenders" are saved to $state{$user} = [ run_number, exp_inc ]
	31	# and removed when they resolve their quota situation. For this
	32	# to work (and emails to get sent), pass --auto (most certain when
	33	# running from cron).
	34	#
	35	# run_number is the run (tick) when they were first spotted to
	36	# be over safe value, and exp_inc is the interval at which they
	37	# need to be reminded. (We use 1, 2, 4, 8, 16...). Each 'tick'
	38	# and number usually refers to 1 day (if ran from cron daily)
	39
	40	# In the current implimentation, folks will be reminded on
	41	# 1st, 3rd, 7th, 15th and 31st day after their quota overrun.
	42	# Then they will be given 2 months of no messages, and then
	43	# after 3 months all together, the "warning cycle" will start again.
	44
	45	# - If user is over 0.96 on mail or db volume, we will add 2% to his
	46	# quota one-time to alleviate immediate problem.
	47
	48	use warnings;
	49	use strict;
	50	use Storable qw/nstore retrieve/;
	51	use Fatal qw/open write read close/;
	52	use Data::Dumper qw/Dumper/;
	53
	54	# Warn threshold
	55	our %threshold = ( user => 0.8, mail => 0.6, db => 0.4 );
	56	# AFS servers with volumes
6d76f213	57	our @servers = qw/gibran/;
63ecf196	58	# Which volumes we want?
	59	# (Your volumes need to be in format (TYPE).USERNAME, otherwise
	60	# you will have to modify below where we take .USERNAME to mean
	61	# person's USERNAME). Changing this will also trigger changes
	62	# in template email (see DATA below).
0cf246e4	63	our $volume_pattern = '(mail\|user)\.[a-zA-Z0-9]+'; # implicit ^...$
63ecf196	64	# Cache file
	65	our $statefile = '/var/cache/hcoop/quotacheck';
	66	# Hash structure containing load of $statefile
	67	our %state;
	68	# Update cache & send out emails (--auto), or just run terminal report
	69	our $auto = grep {/^--auto$/} @ARGV;
	70	# Dump statefile in readable format to terminal and exit (--dump)
	71	our $dump = grep {/^--dump$/} @ARGV;
	72	# Do not send emails:
	73	our $nomail = grep {/^--nomail$/} @ARGV;
	74	# E-mail template that is evaled before sending
	75	our @template = <DATA>;
	76
	77
	78	# WARMUP
	79
	80	# If statefile is there, eventually load it
	81	if ( -e $statefile ) {
	82	if ( $auto or $dump) { %state = %{ retrieve($statefile) } }
	83	}
	84
	85	# Only dump wanted?
	86	if ( $dump ) { print Dumper \%state; exit 0 }
	87
	88	# Increment tick
	89	$state{_tick}++;
	90
	91
	92	# BEGIN WOROK
	93
	94	# Get volume lists from all servers
	95	my @list;
	96	for (@servers) { push @list, `vos listvol $_ -format` }
	97	chomp for @list;
	98
	99	my $current; # name of volume being parsed
	100	my %vol; # will contain hash result of parsing
	101	my %warn; # list of people and data for > $threshold folks
	102	my %nowarn; # same data as in %warn, but for people NOT over threshold
	103
	104	#
	105	# Copy vol from @list (vos listvol output) into %vol hash.
	106	# We read in all values; all values from vol listvol -format
	107	# output for a volume are read in. Such as
	108	# $vol{user.docelic}{type} = 'RW'
	109	#
	110	for (@list) {
	111	my ($parm, $value) = split(/\s+/, $_, 2);
	112	defined $parm and defined $value or next;
	113
	114	if ( $parm eq 'name' ) {
	115	if ( $value =~ /^$volume_pattern$/ ) {
	116	$current = $value
	117	} else {
	118	$current = ''
	119	}
	120	} elsif ($current) {
	121	$vol{$current}{$parm} = $value
	122	}
	123	}
	124
	125	#
	126	# Copy > $threshold people to %warn hash like this:
	127	# $warn{docelic}{mail} = 0.9
128	#
129	while (my($k,$v) = each %vol) {
130	next unless $$v{maxquota};
131	my ($type,$user) = split(/\./, $k);
132
133	my $perc = sprintf('%.3f', $$v{diskused} / $$v{maxquota});
134
135	if ( $perc > $threshold{$type} ) {
136	$warn{$user}{$type} = $perc;
137	}
138
139	# A bit of duplication with %warn, but wth..
140	$nowarn{$user}{$type} = $perc;
141	}
142
143	#
144	# Reset counter for people who solved their quota thing somehow
145	#
146	my @tmp = keys %state;
147	for my $person ( @tmp ) {
148	next if $person eq '_tick';
149	unless ( $warn{$person} ) {
150	print "User $person resolved quota problem\n";
151	delete $state{$person}
152	}
153	}
154
155	#
156	# Go over %warn, print info to terminal, update %state
157	# when needed to create new entry for new quota detections.
158	#
159	while (my($k,$v) = each %warn) {
160	print "$k ";
161	my @line;
162	while (my($k2,$v2) = each %$v) {
163	push @line, "$k2=$v2";
164	}
165	print "@line";
166
167	# Register person in %state
168	if ( ! $state{$k} ) {
169	$state{$k} = [ $state{_tick}, 2 ];
170	}
171
172	print "\n";
173	}
174
175	#
176	# See who's entitled to receiving an email at this point;
177	# that is, go over %state, send emails to people who need
178	# to get it in this turn, and exponentially prolong the time
179	# till next message from us.
180	# (They receive message on 1st, 3rd, 7th, 15th, 31st...)
181	#
182	while (my($user,$state) = each %state ) {
183	next if $user eq '_tick';
184	#print " $$state[0] + $$state[1] == $state{_tick} + 2\n";
185
186	if ( $$state[0] + $$state[1] == $state{_tick} + 2 ) {
187
188	if ($auto and !$nomail) {
189	print "Notifying $user, next notice in $$state[1] days.\n";
190	email_notify($user);
191	}
192
193	$$state[1] *= 2;
194
195	# Following code would re-start the notifications we
196	# send to people after 120 days of not clearing
197	# their issue, but let's keep it disabled for
198	# now.. They get a notice on 127th day, and then
199	# on 255th etc., we don't reset the sequence.
200
201	#$$state[1] >= 120 and do {
202	# $$state[0] = $state{_tick};
203	# $$state[1] = 2;
204	#};
205	}
206	}
207
208
209	# Write cache
210	if ( $auto ) {
211	# Already done above, with also printing a nice
212	# message
213	#for ( keys %warn ) {
214	# delete $nowarn{$_}
215	#}
216	#for ( keys %nowarn) {
217	# delete $state{$_};
218	#}
219
220	nstore \%state, $statefile;
221	}
222
223	DONE:
224	exit 0;
225	##############################################
226	# Helpers below
227
228
229	sub email_notify {
230	my $user = shift;
231
232	my @msg = @template;
233	for ( @msg ) {
234	$_ =~ s/\$vol{(\w+)}{(\w+)}/$vol{"$1.$user"}{$2}/mge;
235	$_ =~ s/\$threshold{(\w+)}/$threshold{$1}/mge;
236	$_ =~ s/\$warn{(\w+)}/$nowarn{$user}{$1}/mge;
237	$_ =~ s/\$CRIT{(\w+)}/$warn{$user}{$1} ? " (APPROACHING QUOTA)" : ""/mge;
238	}
239	#print join('', @msg);
fa44e5b4	240	open(MAIL, "\| mail -s 'Approaching quota limit at HCoop' -a 'From: admins\@hcoop.net' -b 'admins\@hcoop.net' '$user\@hcoop.net'");
63ecf196	241	{ no warnings; print main::MAIL join('', @msg); }
	242	close MAIL;
	243	}
	244
	245
	246	# BEGIN_OF_ENTRY
	247	# name user.docelic
	248	# id 536872079
	249	# serv 69.90.123.67 deleuze.hcoop.net
	250	# part /vicepa
	251	# status OK
	252	# backupID 536872081
	253	# parentID 536872079
	254	# cloneID 536955593
	255	# inUse Y
	256	# needsSalvaged N
	257	# destroyMe N
	258	# type RW
	259	# creationDate 1195841197 Fri Nov 23 13:06:37 2007
	260	# accessDate 0 Wed Dec 31 19:00:00 1969
	261	# updateDate 1198364408 Sat Dec 22 18:00:08 2007
	262	# backupDate 1198322931 Sat Dec 22 06:28:51 2007
	263	# copyDate 1195843557 Fri Nov 23 13:45:57 2007
	264	# flags 0 (Optional)
	265	# diskused 36718
	266	# maxquota 400000
	267	# minquota 0 (Optional)
	268	# filecount 6338
	269	# dayUse 20097
	270	# weekUse 172438 (Optional)
	271	# spare2 0 (Optional)
	272	# spare3 0 (Optional)
	273	# END_OF_ENTRY
	274
	275	__DATA__
	276	Hello,
	277
	278	Some of your data volumes at HCoop are approaching quota
	279	limit.
	280
	281	We use quotas to "keep the shop clean", but we regularly
	282	grant quota increase requests to all members who submit them.
	283
	284	We wouldn't want you hit the quota limit, hence the early
	285	notice. This is especially true for the mail and database
	286	volumes where the data can accumulate "by itself", and where
	287	reaching quota limits may be particularly inconvenient.
	288	If you plan to store more data, it might be a good
	289	time to submit the quota change request through our portal[1].
	290
	291	Here are your volume statistics:
	292
	293	Home:
	294	size=$vol{user}{diskused} KB
	295	quota=$vol{user}{maxquota} KB
	296	threshold=$threshold{user}
	297	use=$warn{user} $CRIT{user}
	298
	299	Mail:
	300	size=$vol{mail}{diskused} KB
	301	quota=$vol{mail}{maxquota} KB
	302	threshold=$threshold{mail}
	303	use=$warn{mail} $CRIT{mail}
	304
63ecf196	305	Please submit your disk quota increase request in a timely
	306	manner at:
	307
	308	[1] https://members.hcoop.net/portal/quota
	309
	310
	311	These automatic notices will be sent to you at exponential
	312	intervals; that means in 2 days, 4 days, 8 days, 16 days and
	313	so on. The method ensures that, if you retain status-quo
	314	(don't submit a quota increase and don't reduce your data size),
	315	you get to receive the reminders less and less often.
	316	When and if the usage percentage of all your volumes drops below
	317	the corresponding quota thresholds, the notifications will stop
	318	and reset.
	319
	320	Thank you,
	321	HCoop
	322