Add 2012 to FSF copyright years for Emacs files (do not merge to trunk)
[bpt/emacs.git] / lisp / gnus / spam-stat.el
CommitLineData
23f87bed
MB
1;;; spam-stat.el --- detecting spam based on statistics
2
49f70d46 3;; Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
23f87bed
MB
4
5;; Author: Alex Schroeder <alex@gnu.org>
6;; Keywords: network
7;; URL: http://www.emacswiki.org/cgi-bin/wiki.pl?SpamStat
8
9;; This file is part of GNU Emacs.
10
5e809f55
GM
11;; GNU Emacs is free software: you can redistribute it and/or modify
12;; it under the terms of the GNU General Public License as published by
13;; the Free Software Foundation, either version 3 of the License, or
14;; (at your option) any later version.
23f87bed 15
5e809f55
GM
16;; GNU Emacs is distributed in the hope that it will be useful,
17;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19;; GNU General Public License for more details.
23f87bed
MB
20
21;; You should have received a copy of the GNU General Public License
5e809f55 22;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
23f87bed
MB
23
24;;; Commentary:
25
26;; This implements spam analysis according to Paul Graham in "A Plan
27;; for Spam". The basis for all this is a statistical distribution of
28;; words for your spam and non-spam mails. We need this information
29;; in a hash-table so that the analysis can use the information when
30;; looking at your mails. Therefore, before you begin, you need tons
31;; of mails (Graham uses 4000 non-spam and 4000 spam mails for his
32;; experiments).
33;;
34;; The main interface to using spam-stat, are the following functions:
35;;
36;; `spam-stat-buffer-is-spam' -- called in a buffer, that buffer is
37;; considered to be a new spam mail; use this for new mail that has
38;; not been processed before
39;;
40;; `spam-stat-buffer-is-non-spam' -- called in a buffer, that buffer
41;; is considered to be a new non-spam mail; use this for new mail that
42;; has not been processed before
43;;
44;; `spam-stat-buffer-change-to-spam' -- called in a buffer, that
45;; buffer is no longer considered to be normal mail but spam; use this
46;; to change the status of a mail that has already been processed as
47;; non-spam
48;;
49;; `spam-stat-buffer-change-to-non-spam' -- called in a buffer, that
50;; buffer is no longer considered to be spam but normal mail; use this
51;; to change the status of a mail that has already been processed as
52;; spam
53;;
54;; `spam-stat-save' -- save the hash table to the file; the filename
55;; used is stored in the variable `spam-stat-file'
56;;
57;; `spam-stat-load' -- load the hash table from a file; the filename
58;; used is stored in the variable `spam-stat-file'
59;;
60;; `spam-stat-score-word' -- return the spam score for a word
61;;
62;; `spam-stat-score-buffer' -- return the spam score for a buffer
63;;
64;; `spam-stat-split-fancy' -- for fancy mail splitting; add
65;; the rule (: spam-stat-split-fancy) to `nnmail-split-fancy'
66;;
67;; This requires the following in your ~/.gnus file:
68;;
69;; (require 'spam-stat)
70;; (spam-stat-load)
71
72;;; Testing:
73
74;; Typical test will involve calls to the following functions:
75;;
76;; Reset: (spam-stat-reset)
77;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
78;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
79;; Save table: (spam-stat-save)
80;; File size: (nth 7 (file-attributes spam-stat-file))
81;; Number of words: (hash-table-count spam-stat)
82;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
83;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
84;; Reduce table size: (spam-stat-reduce-size)
85;; Save table: (spam-stat-save)
86;; File size: (nth 7 (file-attributes spam-stat-file))
87;; Number of words: (hash-table-count spam-stat)
88;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
89;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
90
91;;; Dictionary Creation:
92
93;; Typically, you will filter away mailing lists etc. using specific
94;; rules in `nnmail-split-fancy'. Somewhere among these rules, you
95;; will filter spam. Here is how you would create your dictionary:
96
97;; Reset: (spam-stat-reset)
98;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
99;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
100;; Repeat for any other non-spam group you need...
101;; Reduce table size: (spam-stat-reduce-size)
102;; Save table: (spam-stat-save)
103
104;;; Todo:
105
106;; Speed it up. Integrate with Gnus such that it uses spam and expiry
107;; marks to call the appropriate functions when leaving the summary
108;; buffer and saves the hash table when leaving Gnus. More testing:
109;; More mails, disabling SpamAssassin, double checking algorithm, find
110;; improved algorithm.
111
112;;; Thanks:
113
114;; Ted Zlatanov <tzz@lifelogs.com>
115;; Jesper Harder <harder@myrealbox.com>
116;; Dan Schmidt <dfan@dfan.org>
117;; Lasse Rasinen <lrasinen@iki.fi>
118;; Milan Zamazal <pdm@zamazal.org>
119
120\f
121
122;;; Code:
01c52d31 123(require 'mail-parse)
23f87bed 124
5998e245
JB
125(defvar gnus-original-article-buffer)
126
23f87bed
MB
127(defgroup spam-stat nil
128 "Statistical spam detection for Emacs.
129Use the functions to build a dictionary of words and their statistical
130distribution in spam and non-spam mails. Then use a function to determine
131whether a buffer contains spam or not."
bf247b6e 132 :version "22.1"
23f87bed
MB
133 :group 'gnus)
134
135(defcustom spam-stat-file "~/.spam-stat.el"
136 "File used to save and load the dictionary.
137See `spam-stat-to-hash-table' for the format of the file."
138 :type 'file
139 :group 'spam-stat)
140
141(defcustom spam-stat-install-hooks t
142 "Whether spam-stat should install its hooks in Gnus.
143This is set to nil if you use spam-stat through spam.el."
144 :type 'boolean
145 :group 'spam-stat)
146
147(defcustom spam-stat-unknown-word-score 0.2
148 "The score to use for unknown words.
149Also used for words that don't appear often enough."
150 :type 'number
151 :group 'spam-stat)
152
153(defcustom spam-stat-max-word-length 15
154 "Only words shorter than this will be considered."
155 :type 'integer
156 :group 'spam-stat)
157
158(defcustom spam-stat-max-buffer-length 10240
159 "Only the beginning of buffers will be analyzed.
160This variable says how many characters this will be."
161 :type 'integer
162 :group 'spam-stat)
163
164(defcustom spam-stat-split-fancy-spam-group "mail.spam"
01c52d31
MB
165 "Name of the group where spam should be stored.
166If `spam-stat-split-fancy' is used in fancy splitting rules. Has
167no effect when spam-stat is invoked through spam.el."
23f87bed
MB
168 :type 'string
169 :group 'spam-stat)
170
01c52d31
MB
171(defcustom spam-stat-split-fancy-spam-threshold 0.9
172 "Spam score threshold in spam-stat-split-fancy."
23f87bed
MB
173 :type 'number
174 :group 'spam-stat)
175
01c52d31
MB
176(defcustom spam-stat-washing-hook nil
177 "Hook applied to each message before analysis."
178 :type 'hook
179 :group 'spam-stat)
180
181(defcustom spam-stat-score-buffer-user-functions nil
182 "List of additional scoring functions.
0577a430 183Called one by one on the buffer.
01c52d31
MB
184
185If all of these functions return non-nil answers, these numerical
186answers are added to the computed spam stat score on the buffer. If
187you defun such functions, make sure they don't return the buffer in a
188narrowed state or such: use, for example, `save-excursion'. Each of
189your functions is also passed the initial spam-stat score which might
190aid in your scoring.
191
192Also be careful when defining such functions. If they take a long
193time, they will slow down your mail splitting. Thus, if the buffer is
194large, don't forget to use smaller regions, by wrapping your work in,
195say, `with-spam-stat-max-buffer-size'."
196 :type '(repeat sexp)
197 :group 'spam-stat)
198
199(defcustom spam-stat-process-directory-age 90
200 "Max. age of files to be processed in directory, in days.
201When using `spam-stat-process-spam-directory' or
202`spam-stat-process-non-spam-directory', only files that have
203been touched in this many days will be considered. Without
204this filter, re-training spam-stat with several thousand messages
205will start to take a very long time."
206 :type 'number
207 :group 'spam-stat)
208
209(defvar spam-stat-last-saved-at nil
210 "Time stamp of last change of spam-stat-file on this run")
211
23f87bed
MB
212(defvar spam-stat-syntax-table
213 (let ((table (copy-syntax-table text-mode-syntax-table)))
214 (modify-syntax-entry ?- "w" table)
215 (modify-syntax-entry ?_ "w" table)
216 (modify-syntax-entry ?. "w" table)
217 (modify-syntax-entry ?! "w" table)
218 (modify-syntax-entry ?? "w" table)
219 (modify-syntax-entry ?+ "w" table)
220 table)
221 "Syntax table used when processing mails for statistical analysis.
222The important part is which characters are word constituents.")
223
224(defvar spam-stat-dirty nil
225 "Whether the spam-stat database needs saving.")
226
227(defvar spam-stat-buffer nil
228 "Buffer to use for scoring while splitting.
229This is set by hooking into Gnus.")
230
231(defvar spam-stat-buffer-name " *spam stat buffer*"
232 "Name of the `spam-stat-buffer'.")
233
01c52d31
MB
234(defvar spam-stat-coding-system
235 (if (mm-coding-system-p 'emacs-mule) 'emacs-mule 'raw-text)
236 "Coding system used for `spam-stat-file'.")
23f87bed
MB
237
238;; Hooking into Gnus
239
240(defun spam-stat-store-current-buffer ()
241 "Store a copy of the current buffer in `spam-stat-buffer'."
01c52d31
MB
242 (let ((buf (current-buffer)))
243 (with-current-buffer (get-buffer-create spam-stat-buffer-name)
23f87bed 244 (erase-buffer)
01c52d31 245 (insert-buffer-substring buf)
23f87bed
MB
246 (setq spam-stat-buffer (current-buffer)))))
247
248(defun spam-stat-store-gnus-article-buffer ()
249 "Store a copy of the current article in `spam-stat-buffer'.
250This uses `gnus-article-buffer'."
01c52d31 251 (with-current-buffer gnus-original-article-buffer
23f87bed
MB
252 (spam-stat-store-current-buffer)))
253
254;; Data -- not using defstruct in order to save space and time
255
256(defvar spam-stat (make-hash-table :test 'equal)
257 "Hash table used to store the statistics.
258Use `spam-stat-load' to load the file.
259Every word is used as a key in this table. The value is a vector.
260Use `spam-stat-ngood', `spam-stat-nbad', `spam-stat-good',
261`spam-stat-bad', and `spam-stat-score' to access this vector.")
262
263(defvar spam-stat-ngood 0
264 "The number of good mails in the dictionary.")
265
266(defvar spam-stat-nbad 0
267 "The number of bad mails in the dictionary.")
268
01c52d31
MB
269(defvar spam-stat-error-holder nil
270 "A holder for condition-case errors while scoring buffers.")
271
23f87bed
MB
272(defsubst spam-stat-good (entry)
273 "Return the number of times this word belongs to good mails."
274 (aref entry 0))
275
276(defsubst spam-stat-bad (entry)
277 "Return the number of times this word belongs to bad mails."
278 (aref entry 1))
279
280(defsubst spam-stat-score (entry)
281 "Set the score of this word."
282 (if entry
283 (aref entry 2)
284 spam-stat-unknown-word-score))
285
286(defsubst spam-stat-set-good (entry value)
287 "Set the number of times this word belongs to good mails."
288 (aset entry 0 value))
289
290(defsubst spam-stat-set-bad (entry value)
291 "Set the number of times this word belongs to bad mails."
292 (aset entry 1 value))
293
294(defsubst spam-stat-set-score (entry value)
295 "Set the score of this word."
296 (aset entry 2 value))
297
298(defsubst spam-stat-make-entry (good bad)
299 "Return a vector with the given properties."
300 (let ((entry (vector good bad nil)))
301 (spam-stat-set-score entry (spam-stat-compute-score entry))
302 entry))
303
304;; Computing
305
306(defun spam-stat-compute-score (entry)
307 "Compute the score of this word. 1.0 means spam."
308 ;; promote all numbers to floats for the divisions
309 (let* ((g (* 2.0 (spam-stat-good entry)))
310 (b (float (spam-stat-bad entry))))
311 (cond ((< (+ g b) 5)
312 .2)
313 ((= 0 spam-stat-ngood)
314 .99)
315 ((= 0 spam-stat-nbad)
316 .01)
317 (t
318 (max .01
319 (min .99 (/ (/ b spam-stat-nbad)
320 (+ (/ g spam-stat-ngood)
321 (/ b spam-stat-nbad)))))))))
322
323;; Parsing
324
325(defmacro with-spam-stat-max-buffer-size (&rest body)
01c52d31 326 "Narrow the buffer down to the first 4k characters, then evaluate BODY."
23f87bed
MB
327 `(save-restriction
328 (when (> (- (point-max)
329 (point-min))
330 spam-stat-max-buffer-length)
331 (narrow-to-region (point-min)
332 (+ (point-min) spam-stat-max-buffer-length)))
333 ,@body))
334
335(defun spam-stat-buffer-words ()
caf213c5 336 "Return a hash table of words and number of occurrences in the buffer."
01c52d31 337 (run-hooks 'spam-stat-washing-hook)
23f87bed
MB
338 (with-spam-stat-max-buffer-size
339 (with-syntax-table spam-stat-syntax-table
340 (goto-char (point-min))
341 (let ((result (make-hash-table :test 'equal))
342 word count)
343 (while (re-search-forward "\\w+" nil t)
344 (setq word (match-string-no-properties 0)
345 count (1+ (gethash word result 0)))
346 (when (< (length word) spam-stat-max-word-length)
347 (puthash word count result)))
348 result))))
349
350(defun spam-stat-buffer-is-spam ()
351 "Consider current buffer to be a new spam mail."
352 (setq spam-stat-nbad (1+ spam-stat-nbad))
353 (maphash
354 (lambda (word count)
355 (let ((entry (gethash word spam-stat)))
356 (if entry
357 (spam-stat-set-bad entry (+ count (spam-stat-bad entry)))
358 (setq entry (spam-stat-make-entry 0 count)))
359 (spam-stat-set-score entry (spam-stat-compute-score entry))
360 (puthash word entry spam-stat)))
361 (spam-stat-buffer-words))
362 (setq spam-stat-dirty t))
363
364(defun spam-stat-buffer-is-non-spam ()
365 "Consider current buffer to be a new non-spam mail."
366 (setq spam-stat-ngood (1+ spam-stat-ngood))
367 (maphash
368 (lambda (word count)
369 (let ((entry (gethash word spam-stat)))
370 (if entry
371 (spam-stat-set-good entry (+ count (spam-stat-good entry)))
372 (setq entry (spam-stat-make-entry count 0)))
373 (spam-stat-set-score entry (spam-stat-compute-score entry))
374 (puthash word entry spam-stat)))
375 (spam-stat-buffer-words))
376 (setq spam-stat-dirty t))
377
64c7b653
GM
378(autoload 'gnus-message "gnus-util")
379
23f87bed
MB
380(defun spam-stat-buffer-change-to-spam ()
381 "Consider current buffer no longer normal mail but spam."
382 (setq spam-stat-nbad (1+ spam-stat-nbad)
383 spam-stat-ngood (1- spam-stat-ngood))
384 (maphash
385 (lambda (word count)
386 (let ((entry (gethash word spam-stat)))
387 (if (not entry)
01c52d31 388 (gnus-message 8 "This buffer has unknown words in it")
23f87bed
MB
389 (spam-stat-set-good entry (- (spam-stat-good entry) count))
390 (spam-stat-set-bad entry (+ (spam-stat-bad entry) count))
391 (spam-stat-set-score entry (spam-stat-compute-score entry))
392 (puthash word entry spam-stat))))
393 (spam-stat-buffer-words))
394 (setq spam-stat-dirty t))
395
396(defun spam-stat-buffer-change-to-non-spam ()
397 "Consider current buffer no longer spam but normal mail."
398 (setq spam-stat-nbad (1- spam-stat-nbad)
399 spam-stat-ngood (1+ spam-stat-ngood))
400 (maphash
401 (lambda (word count)
402 (let ((entry (gethash word spam-stat)))
403 (if (not entry)
01c52d31 404 (gnus-message 8 "This buffer has unknown words in it")
23f87bed
MB
405 (spam-stat-set-good entry (+ (spam-stat-good entry) count))
406 (spam-stat-set-bad entry (- (spam-stat-bad entry) count))
407 (spam-stat-set-score entry (spam-stat-compute-score entry))
408 (puthash word entry spam-stat))))
409 (spam-stat-buffer-words))
410 (setq spam-stat-dirty t))
411
412;; Saving and Loading
413
414(defun spam-stat-save (&optional force)
415 "Save the `spam-stat' hash table as lisp file.
416With a prefix argument save unconditionally."
417 (interactive "P")
418 (when (or force spam-stat-dirty)
01c52d31
MB
419 (let ((coding-system-for-write spam-stat-coding-system))
420 (with-temp-file spam-stat-file
421 (let ((standard-output (current-buffer))
422 (font-lock-maximum-size 0))
423 (insert (format ";-*- coding: %s; -*-\n" spam-stat-coding-system))
424 (insert (format "(setq spam-stat-ngood %d spam-stat-nbad %d
425spam-stat (spam-stat-to-hash-table '(" spam-stat-ngood spam-stat-nbad))
426 (maphash (lambda (word entry)
427 (prin1 (list word
428 (spam-stat-good entry)
429 (spam-stat-bad entry))))
430 spam-stat)
431 (insert ")))"))))
432 (message "Saved %s." spam-stat-file)
433 (setq spam-stat-dirty nil
434 spam-stat-last-saved-at (nth 5 (file-attributes spam-stat-file)))))
23f87bed
MB
435
436(defun spam-stat-load ()
437 "Read the `spam-stat' hash table from disk."
438 ;; TODO: maybe we should warn the user if spam-stat-dirty is t?
01c52d31
MB
439 (let ((coding-system-for-read spam-stat-coding-system))
440 (cond (spam-stat-dirty (message "Spam stat not loaded: spam-stat-dirty t"))
441 ((or (not (boundp 'spam-stat-last-saved-at))
442 (null spam-stat-last-saved-at)
443 (not (equal spam-stat-last-saved-at
444 (nth 5 (file-attributes spam-stat-file)))))
0577a430 445 (progn
01c52d31
MB
446 (load-file spam-stat-file)
447 (setq spam-stat-dirty nil
0577a430 448 spam-stat-last-saved-at
01c52d31 449 (nth 5 (file-attributes spam-stat-file)))))
0577a430 450 (t (message "Spam stat file not loaded: no change in disk.")))))
23f87bed
MB
451
452(defun spam-stat-to-hash-table (entries)
453 "Turn list ENTRIES into a hash table and store as `spam-stat'.
454Every element in ENTRIES has the form \(WORD GOOD BAD) where WORD is
455the word string, NGOOD is the number of good mails it has appeared in,
456NBAD is the number of bad mails it has appeared in, GOOD is the number
457of times it appeared in good mails, and BAD is the number of times it
458has appeared in bad mails."
01c52d31
MB
459 (let ((table (make-hash-table :size (length entries)
460 :test 'equal)))
23f87bed
MB
461 (mapc (lambda (l)
462 (puthash (car l)
463 (spam-stat-make-entry (nth 1 l) (nth 2 l))
464 table))
465 entries)
466 table))
467
468(defun spam-stat-reset ()
469 "Reset `spam-stat' to an empty hash-table.
470This deletes all the statistics."
471 (interactive)
472 (setq spam-stat (make-hash-table :test 'equal)
473 spam-stat-ngood 0
474 spam-stat-nbad 0)
475 (setq spam-stat-dirty t))
476
477;; Scoring buffers
478
479(defvar spam-stat-score-data nil
480 "Raw data used in the last run of `spam-stat-score-buffer'.")
481
482(defsubst spam-stat-score-word (word)
483 "Return score for WORD.
484The default score for unknown words is stored in
485`spam-stat-unknown-word-score'."
486 (spam-stat-score (gethash word spam-stat)))
487
488(defun spam-stat-buffer-words-with-scores ()
489 "Process current buffer, return the 15 most conspicuous words.
490These are the words whose spam-stat differs the most from 0.5.
491The list returned contains elements of the form \(WORD SCORE DIFF),
492where DIFF is the difference between SCORE and 0.5."
01c52d31
MB
493 (let (result word score)
494 (maphash (lambda (word ignore)
495 (setq score (spam-stat-score-word word)
496 result (cons (list word score (abs (- score 0.5)))
497 result)))
498 (spam-stat-buffer-words))
499 (setq result (sort result (lambda (a b) (< (nth 2 b) (nth 2 a)))))
500 (setcdr (nthcdr 14 result) nil)
501 result))
23f87bed
MB
502
503(defun spam-stat-score-buffer ()
01c52d31
MB
504 "Return a score describing the spam-probability for this buffer.
505Add user supplied modifications if supplied."
0577a430 506 (interactive) ; helps in debugging.
23f87bed 507 (setq spam-stat-score-data (spam-stat-buffer-words-with-scores))
01c52d31
MB
508 (let* ((probs (mapcar 'cadr spam-stat-score-data))
509 (prod (apply #'* probs))
0577a430 510 (score0
01c52d31
MB
511 (/ prod (+ prod (apply #'* (mapcar #'(lambda (x) (- 1 x))
512 probs)))))
513 (score1s
514 (condition-case
515 spam-stat-error-holder
516 (spam-stat-score-buffer-user score0)
517 (error nil)))
518 (ans
519 (if score1s (+ score0 score1s) score0)))
0577a430 520 (when (interactive-p)
01c52d31
MB
521 (message "%S" ans))
522 ans))
523
524(defun spam-stat-score-buffer-user (&rest args)
525 (let* ((scores
0577a430
JB
526 (mapcar
527 (lambda (fn)
01c52d31
MB
528 (apply fn args))
529 spam-stat-score-buffer-user-functions)))
0577a430 530 (if (memq nil scores) nil
01c52d31 531 (apply #'+ scores))))
23f87bed
MB
532
533(defun spam-stat-split-fancy ()
534 "Return the name of the spam group if the current mail is spam.
535Use this function on `nnmail-split-fancy'. If you are interested in
536the raw data used for the last run of `spam-stat-score-buffer',
537check the variable `spam-stat-score-data'."
01c52d31 538 (condition-case spam-stat-error-holder
23f87bed
MB
539 (progn
540 (set-buffer spam-stat-buffer)
541 (goto-char (point-min))
01c52d31 542 (when (> (spam-stat-score-buffer) spam-stat-split-fancy-spam-threshold)
23f87bed
MB
543 (when (boundp 'nnmail-split-trace)
544 (mapc (lambda (entry)
545 (push entry nnmail-split-trace))
546 spam-stat-score-data))
547 spam-stat-split-fancy-spam-group))
01c52d31 548 (error (message "Error in spam-stat-split-fancy: %S" spam-stat-error-holder)
23f87bed
MB
549 nil)))
550
551;; Testing
552
01c52d31 553(defun spam-stat-strip-xref ()
84b5fcf6 554 "Strip the Xref header."
01c52d31
MB
555 (save-restriction
556 (mail-narrow-to-head)
557 (when (re-search-forward "^Xref:.*\n" nil t)
558 (delete-region (match-beginning 0) (match-end 0)))))
559
23f87bed
MB
560(defun spam-stat-process-directory (dir func)
561 "Process all the regular files in directory DIR using function FUNC."
562 (let* ((files (directory-files dir t "^[^.]"))
563 (max (/ (length files) 100.0))
564 (count 0))
565 (with-temp-buffer
566 (dolist (f files)
567 (when (and (file-readable-p f)
568 (file-regular-p f)
01c52d31
MB
569 (> (nth 7 (file-attributes f)) 0)
570 (< (time-to-number-of-days (time-since (nth 5 (file-attributes f))))
571 spam-stat-process-directory-age))
23f87bed
MB
572 (setq count (1+ count))
573 (message "Reading %s: %.2f%%" dir (/ count max))
01c52d31
MB
574 (insert-file-contents-literally f)
575 (spam-stat-strip-xref)
23f87bed
MB
576 (funcall func)
577 (erase-buffer))))))
578
579(defun spam-stat-process-spam-directory (dir)
580 "Process all the regular files in directory DIR as spam."
581 (interactive "D")
582 (spam-stat-process-directory dir 'spam-stat-buffer-is-spam))
583
584(defun spam-stat-process-non-spam-directory (dir)
585 "Process all the regular files in directory DIR as non-spam."
586 (interactive "D")
587 (spam-stat-process-directory dir 'spam-stat-buffer-is-non-spam))
588
589(defun spam-stat-count ()
590 "Return size of `spam-stat'."
591 (interactive)
592 (hash-table-count spam-stat))
593
01c52d31 594(defun spam-stat-test-directory (dir &optional verbose)
23f87bed
MB
595 "Test all the regular files in directory DIR for spam.
596If the result is 1.0, then all files are considered spam.
597If the result is 0.0, non of the files is considered spam.
01c52d31
MB
598You can use this to determine error rates.
599
600If VERBOSE is non-nil display names of files detected as spam or
601non-spam in a temporary buffer. If it is the symbol `ham',
602display non-spam files; otherwise display spam files."
603 (interactive "DDirectory: ")
23f87bed 604 (let* ((files (directory-files dir t "^[^.]"))
01c52d31
MB
605 display-files
606 buffer-score
23f87bed
MB
607 (total (length files))
608 (score 0.0); float
609 (max (/ total 100.0)); float
610 (count 0))
611 (with-temp-buffer
612 (dolist (f files)
613 (when (and (file-readable-p f)
614 (file-regular-p f)
615 (> (nth 7 (file-attributes f)) 0))
616 (setq count (1+ count))
01c52d31
MB
617 (message "Reading %.2f%%, score %.2f"
618 (/ count max) (/ score count))
619 (insert-file-contents-literally f)
620 (setq buffer-score (spam-stat-score-buffer))
621 (when (> buffer-score 0.9)
23f87bed 622 (setq score (1+ score)))
01c52d31
MB
623 (when verbose
624 (if (> buffer-score 0.9)
625 (unless (eq verbose 'ham) (push f display-files))
626 (when (eq verbose 'ham) (push f display-files))))
23f87bed 627 (erase-buffer))))
01c52d31
MB
628 (when display-files
629 (with-output-to-temp-buffer "*spam-stat results*"
630 (dolist (file display-files)
631 (princ file)
632 (terpri))))
23f87bed
MB
633 (message "Final score: %d / %d = %f" score total (/ score total))))
634
635;; Shrinking the dictionary
636
637(defun spam-stat-reduce-size (&optional count)
638 "Reduce the size of `spam-stat'.
639This removes all words that occur less than COUNT from the dictionary.
640COUNT defaults to 5"
641 (interactive)
642 (setq count (or count 5))
643 (maphash (lambda (key entry)
644 (when (< (+ (spam-stat-good entry)
645 (spam-stat-bad entry))
646 count)
647 (remhash key spam-stat)))
648 spam-stat)
649 (setq spam-stat-dirty t))
650
651(defun spam-stat-install-hooks-function ()
01c52d31 652 "Install the spam-stat function hooks."
23f87bed
MB
653 (interactive)
654 (add-hook 'nnmail-prepare-incoming-message-hook
655 'spam-stat-store-current-buffer)
656 (add-hook 'gnus-select-article-hook
657 'spam-stat-store-gnus-article-buffer))
658
659(when spam-stat-install-hooks
660 (spam-stat-install-hooks-function))
661
662(defun spam-stat-unload-hook ()
01c52d31 663 "Uninstall the spam-stat function hooks."
23f87bed
MB
664 (interactive)
665 (remove-hook 'nnmail-prepare-incoming-message-hook
666 'spam-stat-store-current-buffer)
667 (remove-hook 'gnus-select-article-hook
668 'spam-stat-store-gnus-article-buffer))
669
e0281050
RS
670(add-hook 'spam-stat-unload-hook 'spam-stat-unload-hook)
671
23f87bed
MB
672(provide 'spam-stat)
673
cbee283d 674;; arch-tag: ff1d2200-8ddb-42fb-bb7b-1b5e20448554
23f87bed 675;;; spam-stat.el ends here