Fix up comment convention on the arch-tag lines.
[bpt/emacs.git] / lisp / gnus / spam-stat.el
CommitLineData
23f87bed
MB
1;;; spam-stat.el --- detecting spam based on statistics
2
e3fe4da0 3;; Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
23f87bed
MB
4
5;; Author: Alex Schroeder <alex@gnu.org>
6;; Keywords: network
7;; URL: http://www.emacswiki.org/cgi-bin/wiki.pl?SpamStat
8
9;; This file is part of GNU Emacs.
10
11;; This is free software; you can redistribute it and/or modify it
12;; under the terms of the GNU General Public License as published by
5a9dffec 13;; the Free Software Foundation; either version 3, or (at your option)
23f87bed
MB
14;; any later version.
15
16;; This is distributed in the hope that it will be useful, but WITHOUT
17;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18;; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
19;; License for more details.
20
21;; You should have received a copy of the GNU General Public License
22;; along with GNU Emacs; see the file COPYING. If not, write to the
3a35cf56
LK
23;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
24;; Boston, MA 02110-1301, USA.
23f87bed
MB
25
26;;; Commentary:
27
28;; This implements spam analysis according to Paul Graham in "A Plan
29;; for Spam". The basis for all this is a statistical distribution of
30;; words for your spam and non-spam mails. We need this information
31;; in a hash-table so that the analysis can use the information when
32;; looking at your mails. Therefore, before you begin, you need tons
33;; of mails (Graham uses 4000 non-spam and 4000 spam mails for his
34;; experiments).
35;;
36;; The main interface to using spam-stat, are the following functions:
37;;
38;; `spam-stat-buffer-is-spam' -- called in a buffer, that buffer is
39;; considered to be a new spam mail; use this for new mail that has
40;; not been processed before
41;;
42;; `spam-stat-buffer-is-non-spam' -- called in a buffer, that buffer
43;; is considered to be a new non-spam mail; use this for new mail that
44;; has not been processed before
45;;
46;; `spam-stat-buffer-change-to-spam' -- called in a buffer, that
47;; buffer is no longer considered to be normal mail but spam; use this
48;; to change the status of a mail that has already been processed as
49;; non-spam
50;;
51;; `spam-stat-buffer-change-to-non-spam' -- called in a buffer, that
52;; buffer is no longer considered to be spam but normal mail; use this
53;; to change the status of a mail that has already been processed as
54;; spam
55;;
56;; `spam-stat-save' -- save the hash table to the file; the filename
57;; used is stored in the variable `spam-stat-file'
58;;
59;; `spam-stat-load' -- load the hash table from a file; the filename
60;; used is stored in the variable `spam-stat-file'
61;;
62;; `spam-stat-score-word' -- return the spam score for a word
63;;
64;; `spam-stat-score-buffer' -- return the spam score for a buffer
65;;
66;; `spam-stat-split-fancy' -- for fancy mail splitting; add
67;; the rule (: spam-stat-split-fancy) to `nnmail-split-fancy'
68;;
69;; This requires the following in your ~/.gnus file:
70;;
71;; (require 'spam-stat)
72;; (spam-stat-load)
73
74;;; Testing:
75
76;; Typical test will involve calls to the following functions:
77;;
78;; Reset: (spam-stat-reset)
79;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
80;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
81;; Save table: (spam-stat-save)
82;; File size: (nth 7 (file-attributes spam-stat-file))
83;; Number of words: (hash-table-count spam-stat)
84;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
85;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
86;; Reduce table size: (spam-stat-reduce-size)
87;; Save table: (spam-stat-save)
88;; File size: (nth 7 (file-attributes spam-stat-file))
89;; Number of words: (hash-table-count spam-stat)
90;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
91;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
92
93;;; Dictionary Creation:
94
95;; Typically, you will filter away mailing lists etc. using specific
96;; rules in `nnmail-split-fancy'. Somewhere among these rules, you
97;; will filter spam. Here is how you would create your dictionary:
98
99;; Reset: (spam-stat-reset)
100;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
101;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
102;; Repeat for any other non-spam group you need...
103;; Reduce table size: (spam-stat-reduce-size)
104;; Save table: (spam-stat-save)
105
106;;; Todo:
107
108;; Speed it up. Integrate with Gnus such that it uses spam and expiry
109;; marks to call the appropriate functions when leaving the summary
110;; buffer and saves the hash table when leaving Gnus. More testing:
111;; More mails, disabling SpamAssassin, double checking algorithm, find
112;; improved algorithm.
113
114;;; Thanks:
115
116;; Ted Zlatanov <tzz@lifelogs.com>
117;; Jesper Harder <harder@myrealbox.com>
118;; Dan Schmidt <dfan@dfan.org>
119;; Lasse Rasinen <lrasinen@iki.fi>
120;; Milan Zamazal <pdm@zamazal.org>
121
122\f
123
124;;; Code:
01c52d31 125(require 'mail-parse)
23f87bed 126
5998e245
JB
127(defvar gnus-original-article-buffer)
128
23f87bed
MB
129(defgroup spam-stat nil
130 "Statistical spam detection for Emacs.
131Use the functions to build a dictionary of words and their statistical
132distribution in spam and non-spam mails. Then use a function to determine
133whether a buffer contains spam or not."
bf247b6e 134 :version "22.1"
23f87bed
MB
135 :group 'gnus)
136
137(defcustom spam-stat-file "~/.spam-stat.el"
138 "File used to save and load the dictionary.
139See `spam-stat-to-hash-table' for the format of the file."
140 :type 'file
141 :group 'spam-stat)
142
143(defcustom spam-stat-install-hooks t
144 "Whether spam-stat should install its hooks in Gnus.
145This is set to nil if you use spam-stat through spam.el."
146 :type 'boolean
147 :group 'spam-stat)
148
149(defcustom spam-stat-unknown-word-score 0.2
150 "The score to use for unknown words.
151Also used for words that don't appear often enough."
152 :type 'number
153 :group 'spam-stat)
154
155(defcustom spam-stat-max-word-length 15
156 "Only words shorter than this will be considered."
157 :type 'integer
158 :group 'spam-stat)
159
160(defcustom spam-stat-max-buffer-length 10240
161 "Only the beginning of buffers will be analyzed.
162This variable says how many characters this will be."
163 :type 'integer
164 :group 'spam-stat)
165
166(defcustom spam-stat-split-fancy-spam-group "mail.spam"
01c52d31
MB
167 "Name of the group where spam should be stored.
168If `spam-stat-split-fancy' is used in fancy splitting rules. Has
169no effect when spam-stat is invoked through spam.el."
23f87bed
MB
170 :type 'string
171 :group 'spam-stat)
172
01c52d31
MB
173(defcustom spam-stat-split-fancy-spam-threshold 0.9
174 "Spam score threshold in spam-stat-split-fancy."
23f87bed
MB
175 :type 'number
176 :group 'spam-stat)
177
01c52d31
MB
178(defcustom spam-stat-washing-hook nil
179 "Hook applied to each message before analysis."
180 :type 'hook
181 :group 'spam-stat)
182
183(defcustom spam-stat-score-buffer-user-functions nil
184 "List of additional scoring functions.
185Called one by one on the buffer.
186
187If all of these functions return non-nil answers, these numerical
188answers are added to the computed spam stat score on the buffer. If
189you defun such functions, make sure they don't return the buffer in a
190narrowed state or such: use, for example, `save-excursion'. Each of
191your functions is also passed the initial spam-stat score which might
192aid in your scoring.
193
194Also be careful when defining such functions. If they take a long
195time, they will slow down your mail splitting. Thus, if the buffer is
196large, don't forget to use smaller regions, by wrapping your work in,
197say, `with-spam-stat-max-buffer-size'."
198 :type '(repeat sexp)
199 :group 'spam-stat)
200
201(defcustom spam-stat-process-directory-age 90
202 "Max. age of files to be processed in directory, in days.
203When using `spam-stat-process-spam-directory' or
204`spam-stat-process-non-spam-directory', only files that have
205been touched in this many days will be considered. Without
206this filter, re-training spam-stat with several thousand messages
207will start to take a very long time."
208 :type 'number
209 :group 'spam-stat)
210
211(defvar spam-stat-last-saved-at nil
212 "Time stamp of last change of spam-stat-file on this run")
213
23f87bed
MB
214(defvar spam-stat-syntax-table
215 (let ((table (copy-syntax-table text-mode-syntax-table)))
216 (modify-syntax-entry ?- "w" table)
217 (modify-syntax-entry ?_ "w" table)
218 (modify-syntax-entry ?. "w" table)
219 (modify-syntax-entry ?! "w" table)
220 (modify-syntax-entry ?? "w" table)
221 (modify-syntax-entry ?+ "w" table)
222 table)
223 "Syntax table used when processing mails for statistical analysis.
224The important part is which characters are word constituents.")
225
226(defvar spam-stat-dirty nil
227 "Whether the spam-stat database needs saving.")
228
229(defvar spam-stat-buffer nil
230 "Buffer to use for scoring while splitting.
231This is set by hooking into Gnus.")
232
233(defvar spam-stat-buffer-name " *spam stat buffer*"
234 "Name of the `spam-stat-buffer'.")
235
01c52d31
MB
236(defvar spam-stat-coding-system
237 (if (mm-coding-system-p 'emacs-mule) 'emacs-mule 'raw-text)
238 "Coding system used for `spam-stat-file'.")
23f87bed
MB
239
240;; Hooking into Gnus
241
242(defun spam-stat-store-current-buffer ()
243 "Store a copy of the current buffer in `spam-stat-buffer'."
01c52d31
MB
244 (let ((buf (current-buffer)))
245 (with-current-buffer (get-buffer-create spam-stat-buffer-name)
23f87bed 246 (erase-buffer)
01c52d31 247 (insert-buffer-substring buf)
23f87bed
MB
248 (setq spam-stat-buffer (current-buffer)))))
249
250(defun spam-stat-store-gnus-article-buffer ()
251 "Store a copy of the current article in `spam-stat-buffer'.
252This uses `gnus-article-buffer'."
01c52d31 253 (with-current-buffer gnus-original-article-buffer
23f87bed
MB
254 (spam-stat-store-current-buffer)))
255
256;; Data -- not using defstruct in order to save space and time
257
258(defvar spam-stat (make-hash-table :test 'equal)
259 "Hash table used to store the statistics.
260Use `spam-stat-load' to load the file.
261Every word is used as a key in this table. The value is a vector.
262Use `spam-stat-ngood', `spam-stat-nbad', `spam-stat-good',
263`spam-stat-bad', and `spam-stat-score' to access this vector.")
264
265(defvar spam-stat-ngood 0
266 "The number of good mails in the dictionary.")
267
268(defvar spam-stat-nbad 0
269 "The number of bad mails in the dictionary.")
270
01c52d31
MB
271(defvar spam-stat-error-holder nil
272 "A holder for condition-case errors while scoring buffers.")
273
23f87bed
MB
274(defsubst spam-stat-good (entry)
275 "Return the number of times this word belongs to good mails."
276 (aref entry 0))
277
278(defsubst spam-stat-bad (entry)
279 "Return the number of times this word belongs to bad mails."
280 (aref entry 1))
281
282(defsubst spam-stat-score (entry)
283 "Set the score of this word."
284 (if entry
285 (aref entry 2)
286 spam-stat-unknown-word-score))
287
288(defsubst spam-stat-set-good (entry value)
289 "Set the number of times this word belongs to good mails."
290 (aset entry 0 value))
291
292(defsubst spam-stat-set-bad (entry value)
293 "Set the number of times this word belongs to bad mails."
294 (aset entry 1 value))
295
296(defsubst spam-stat-set-score (entry value)
297 "Set the score of this word."
298 (aset entry 2 value))
299
300(defsubst spam-stat-make-entry (good bad)
301 "Return a vector with the given properties."
302 (let ((entry (vector good bad nil)))
303 (spam-stat-set-score entry (spam-stat-compute-score entry))
304 entry))
305
306;; Computing
307
308(defun spam-stat-compute-score (entry)
309 "Compute the score of this word. 1.0 means spam."
310 ;; promote all numbers to floats for the divisions
311 (let* ((g (* 2.0 (spam-stat-good entry)))
312 (b (float (spam-stat-bad entry))))
313 (cond ((< (+ g b) 5)
314 .2)
315 ((= 0 spam-stat-ngood)
316 .99)
317 ((= 0 spam-stat-nbad)
318 .01)
319 (t
320 (max .01
321 (min .99 (/ (/ b spam-stat-nbad)
322 (+ (/ g spam-stat-ngood)
323 (/ b spam-stat-nbad)))))))))
324
325;; Parsing
326
327(defmacro with-spam-stat-max-buffer-size (&rest body)
01c52d31 328 "Narrow the buffer down to the first 4k characters, then evaluate BODY."
23f87bed
MB
329 `(save-restriction
330 (when (> (- (point-max)
331 (point-min))
332 spam-stat-max-buffer-length)
333 (narrow-to-region (point-min)
334 (+ (point-min) spam-stat-max-buffer-length)))
335 ,@body))
336
337(defun spam-stat-buffer-words ()
caf213c5 338 "Return a hash table of words and number of occurrences in the buffer."
01c52d31 339 (run-hooks 'spam-stat-washing-hook)
23f87bed
MB
340 (with-spam-stat-max-buffer-size
341 (with-syntax-table spam-stat-syntax-table
342 (goto-char (point-min))
343 (let ((result (make-hash-table :test 'equal))
344 word count)
345 (while (re-search-forward "\\w+" nil t)
346 (setq word (match-string-no-properties 0)
347 count (1+ (gethash word result 0)))
348 (when (< (length word) spam-stat-max-word-length)
349 (puthash word count result)))
350 result))))
351
352(defun spam-stat-buffer-is-spam ()
353 "Consider current buffer to be a new spam mail."
354 (setq spam-stat-nbad (1+ spam-stat-nbad))
355 (maphash
356 (lambda (word count)
357 (let ((entry (gethash word spam-stat)))
358 (if entry
359 (spam-stat-set-bad entry (+ count (spam-stat-bad entry)))
360 (setq entry (spam-stat-make-entry 0 count)))
361 (spam-stat-set-score entry (spam-stat-compute-score entry))
362 (puthash word entry spam-stat)))
363 (spam-stat-buffer-words))
364 (setq spam-stat-dirty t))
365
366(defun spam-stat-buffer-is-non-spam ()
367 "Consider current buffer to be a new non-spam mail."
368 (setq spam-stat-ngood (1+ spam-stat-ngood))
369 (maphash
370 (lambda (word count)
371 (let ((entry (gethash word spam-stat)))
372 (if entry
373 (spam-stat-set-good entry (+ count (spam-stat-good entry)))
374 (setq entry (spam-stat-make-entry count 0)))
375 (spam-stat-set-score entry (spam-stat-compute-score entry))
376 (puthash word entry spam-stat)))
377 (spam-stat-buffer-words))
378 (setq spam-stat-dirty t))
379
64c7b653
GM
380(autoload 'gnus-message "gnus-util")
381
23f87bed
MB
382(defun spam-stat-buffer-change-to-spam ()
383 "Consider current buffer no longer normal mail but spam."
384 (setq spam-stat-nbad (1+ spam-stat-nbad)
385 spam-stat-ngood (1- spam-stat-ngood))
386 (maphash
387 (lambda (word count)
388 (let ((entry (gethash word spam-stat)))
389 (if (not entry)
01c52d31 390 (gnus-message 8 "This buffer has unknown words in it")
23f87bed
MB
391 (spam-stat-set-good entry (- (spam-stat-good entry) count))
392 (spam-stat-set-bad entry (+ (spam-stat-bad entry) count))
393 (spam-stat-set-score entry (spam-stat-compute-score entry))
394 (puthash word entry spam-stat))))
395 (spam-stat-buffer-words))
396 (setq spam-stat-dirty t))
397
398(defun spam-stat-buffer-change-to-non-spam ()
399 "Consider current buffer no longer spam but normal mail."
400 (setq spam-stat-nbad (1- spam-stat-nbad)
401 spam-stat-ngood (1+ spam-stat-ngood))
402 (maphash
403 (lambda (word count)
404 (let ((entry (gethash word spam-stat)))
405 (if (not entry)
01c52d31 406 (gnus-message 8 "This buffer has unknown words in it")
23f87bed
MB
407 (spam-stat-set-good entry (+ (spam-stat-good entry) count))
408 (spam-stat-set-bad entry (- (spam-stat-bad entry) count))
409 (spam-stat-set-score entry (spam-stat-compute-score entry))
410 (puthash word entry spam-stat))))
411 (spam-stat-buffer-words))
412 (setq spam-stat-dirty t))
413
414;; Saving and Loading
415
416(defun spam-stat-save (&optional force)
417 "Save the `spam-stat' hash table as lisp file.
418With a prefix argument save unconditionally."
419 (interactive "P")
420 (when (or force spam-stat-dirty)
01c52d31
MB
421 (let ((coding-system-for-write spam-stat-coding-system))
422 (with-temp-file spam-stat-file
423 (let ((standard-output (current-buffer))
424 (font-lock-maximum-size 0))
425 (insert (format ";-*- coding: %s; -*-\n" spam-stat-coding-system))
426 (insert (format "(setq spam-stat-ngood %d spam-stat-nbad %d
427spam-stat (spam-stat-to-hash-table '(" spam-stat-ngood spam-stat-nbad))
428 (maphash (lambda (word entry)
429 (prin1 (list word
430 (spam-stat-good entry)
431 (spam-stat-bad entry))))
432 spam-stat)
433 (insert ")))"))))
434 (message "Saved %s." spam-stat-file)
435 (setq spam-stat-dirty nil
436 spam-stat-last-saved-at (nth 5 (file-attributes spam-stat-file)))))
23f87bed
MB
437
438(defun spam-stat-load ()
439 "Read the `spam-stat' hash table from disk."
440 ;; TODO: maybe we should warn the user if spam-stat-dirty is t?
01c52d31
MB
441 (let ((coding-system-for-read spam-stat-coding-system))
442 (cond (spam-stat-dirty (message "Spam stat not loaded: spam-stat-dirty t"))
443 ((or (not (boundp 'spam-stat-last-saved-at))
444 (null spam-stat-last-saved-at)
445 (not (equal spam-stat-last-saved-at
446 (nth 5 (file-attributes spam-stat-file)))))
447 (progn
448 (load-file spam-stat-file)
449 (setq spam-stat-dirty nil
450 spam-stat-last-saved-at
451 (nth 5 (file-attributes spam-stat-file)))))
452 (t (message "Spam stat file not loaded: no change in disk..")))))
23f87bed
MB
453
454(defun spam-stat-to-hash-table (entries)
455 "Turn list ENTRIES into a hash table and store as `spam-stat'.
456Every element in ENTRIES has the form \(WORD GOOD BAD) where WORD is
457the word string, NGOOD is the number of good mails it has appeared in,
458NBAD is the number of bad mails it has appeared in, GOOD is the number
459of times it appeared in good mails, and BAD is the number of times it
460has appeared in bad mails."
01c52d31
MB
461 (let ((table (make-hash-table :size (length entries)
462 :test 'equal)))
23f87bed
MB
463 (mapc (lambda (l)
464 (puthash (car l)
465 (spam-stat-make-entry (nth 1 l) (nth 2 l))
466 table))
467 entries)
468 table))
469
470(defun spam-stat-reset ()
471 "Reset `spam-stat' to an empty hash-table.
472This deletes all the statistics."
473 (interactive)
474 (setq spam-stat (make-hash-table :test 'equal)
475 spam-stat-ngood 0
476 spam-stat-nbad 0)
477 (setq spam-stat-dirty t))
478
479;; Scoring buffers
480
481(defvar spam-stat-score-data nil
482 "Raw data used in the last run of `spam-stat-score-buffer'.")
483
484(defsubst spam-stat-score-word (word)
485 "Return score for WORD.
486The default score for unknown words is stored in
487`spam-stat-unknown-word-score'."
488 (spam-stat-score (gethash word spam-stat)))
489
490(defun spam-stat-buffer-words-with-scores ()
491 "Process current buffer, return the 15 most conspicuous words.
492These are the words whose spam-stat differs the most from 0.5.
493The list returned contains elements of the form \(WORD SCORE DIFF),
494where DIFF is the difference between SCORE and 0.5."
01c52d31
MB
495 (let (result word score)
496 (maphash (lambda (word ignore)
497 (setq score (spam-stat-score-word word)
498 result (cons (list word score (abs (- score 0.5)))
499 result)))
500 (spam-stat-buffer-words))
501 (setq result (sort result (lambda (a b) (< (nth 2 b) (nth 2 a)))))
502 (setcdr (nthcdr 14 result) nil)
503 result))
23f87bed
MB
504
505(defun spam-stat-score-buffer ()
01c52d31
MB
506 "Return a score describing the spam-probability for this buffer.
507Add user supplied modifications if supplied."
508 (interactive) ; helps in debugging.
23f87bed 509 (setq spam-stat-score-data (spam-stat-buffer-words-with-scores))
01c52d31
MB
510 (let* ((probs (mapcar 'cadr spam-stat-score-data))
511 (prod (apply #'* probs))
512 (score0
513 (/ prod (+ prod (apply #'* (mapcar #'(lambda (x) (- 1 x))
514 probs)))))
515 (score1s
516 (condition-case
517 spam-stat-error-holder
518 (spam-stat-score-buffer-user score0)
519 (error nil)))
520 (ans
521 (if score1s (+ score0 score1s) score0)))
522 (when (interactive-p)
523 (message "%S" ans))
524 ans))
525
526(defun spam-stat-score-buffer-user (&rest args)
527 (let* ((scores
528 (mapcar
529 (lambda (fn)
530 (apply fn args))
531 spam-stat-score-buffer-user-functions)))
532 (if (memq nil scores) nil
533 (apply #'+ scores))))
23f87bed
MB
534
535(defun spam-stat-split-fancy ()
536 "Return the name of the spam group if the current mail is spam.
537Use this function on `nnmail-split-fancy'. If you are interested in
538the raw data used for the last run of `spam-stat-score-buffer',
539check the variable `spam-stat-score-data'."
01c52d31 540 (condition-case spam-stat-error-holder
23f87bed
MB
541 (progn
542 (set-buffer spam-stat-buffer)
543 (goto-char (point-min))
01c52d31 544 (when (> (spam-stat-score-buffer) spam-stat-split-fancy-spam-threshold)
23f87bed
MB
545 (when (boundp 'nnmail-split-trace)
546 (mapc (lambda (entry)
547 (push entry nnmail-split-trace))
548 spam-stat-score-data))
549 spam-stat-split-fancy-spam-group))
01c52d31 550 (error (message "Error in spam-stat-split-fancy: %S" spam-stat-error-holder)
23f87bed
MB
551 nil)))
552
553;; Testing
554
01c52d31
MB
555(defun spam-stat-strip-xref ()
556 "Strip the the Xref header."
557 (save-restriction
558 (mail-narrow-to-head)
559 (when (re-search-forward "^Xref:.*\n" nil t)
560 (delete-region (match-beginning 0) (match-end 0)))))
561
23f87bed
MB
562(defun spam-stat-process-directory (dir func)
563 "Process all the regular files in directory DIR using function FUNC."
564 (let* ((files (directory-files dir t "^[^.]"))
565 (max (/ (length files) 100.0))
566 (count 0))
567 (with-temp-buffer
568 (dolist (f files)
569 (when (and (file-readable-p f)
570 (file-regular-p f)
01c52d31
MB
571 (> (nth 7 (file-attributes f)) 0)
572 (< (time-to-number-of-days (time-since (nth 5 (file-attributes f))))
573 spam-stat-process-directory-age))
23f87bed
MB
574 (setq count (1+ count))
575 (message "Reading %s: %.2f%%" dir (/ count max))
01c52d31
MB
576 (insert-file-contents-literally f)
577 (spam-stat-strip-xref)
23f87bed
MB
578 (funcall func)
579 (erase-buffer))))))
580
581(defun spam-stat-process-spam-directory (dir)
582 "Process all the regular files in directory DIR as spam."
583 (interactive "D")
584 (spam-stat-process-directory dir 'spam-stat-buffer-is-spam))
585
586(defun spam-stat-process-non-spam-directory (dir)
587 "Process all the regular files in directory DIR as non-spam."
588 (interactive "D")
589 (spam-stat-process-directory dir 'spam-stat-buffer-is-non-spam))
590
591(defun spam-stat-count ()
592 "Return size of `spam-stat'."
593 (interactive)
594 (hash-table-count spam-stat))
595
01c52d31 596(defun spam-stat-test-directory (dir &optional verbose)
23f87bed
MB
597 "Test all the regular files in directory DIR for spam.
598If the result is 1.0, then all files are considered spam.
599If the result is 0.0, non of the files is considered spam.
01c52d31
MB
600You can use this to determine error rates.
601
602If VERBOSE is non-nil display names of files detected as spam or
603non-spam in a temporary buffer. If it is the symbol `ham',
604display non-spam files; otherwise display spam files."
605 (interactive "DDirectory: ")
23f87bed 606 (let* ((files (directory-files dir t "^[^.]"))
01c52d31
MB
607 display-files
608 buffer-score
23f87bed
MB
609 (total (length files))
610 (score 0.0); float
611 (max (/ total 100.0)); float
612 (count 0))
613 (with-temp-buffer
614 (dolist (f files)
615 (when (and (file-readable-p f)
616 (file-regular-p f)
617 (> (nth 7 (file-attributes f)) 0))
618 (setq count (1+ count))
01c52d31
MB
619 (message "Reading %.2f%%, score %.2f"
620 (/ count max) (/ score count))
621 (insert-file-contents-literally f)
622 (setq buffer-score (spam-stat-score-buffer))
623 (when (> buffer-score 0.9)
23f87bed 624 (setq score (1+ score)))
01c52d31
MB
625 (when verbose
626 (if (> buffer-score 0.9)
627 (unless (eq verbose 'ham) (push f display-files))
628 (when (eq verbose 'ham) (push f display-files))))
23f87bed 629 (erase-buffer))))
01c52d31
MB
630 (when display-files
631 (with-output-to-temp-buffer "*spam-stat results*"
632 (dolist (file display-files)
633 (princ file)
634 (terpri))))
23f87bed
MB
635 (message "Final score: %d / %d = %f" score total (/ score total))))
636
637;; Shrinking the dictionary
638
639(defun spam-stat-reduce-size (&optional count)
640 "Reduce the size of `spam-stat'.
641This removes all words that occur less than COUNT from the dictionary.
642COUNT defaults to 5"
643 (interactive)
644 (setq count (or count 5))
645 (maphash (lambda (key entry)
646 (when (< (+ (spam-stat-good entry)
647 (spam-stat-bad entry))
648 count)
649 (remhash key spam-stat)))
650 spam-stat)
651 (setq spam-stat-dirty t))
652
653(defun spam-stat-install-hooks-function ()
01c52d31 654 "Install the spam-stat function hooks."
23f87bed
MB
655 (interactive)
656 (add-hook 'nnmail-prepare-incoming-message-hook
657 'spam-stat-store-current-buffer)
658 (add-hook 'gnus-select-article-hook
659 'spam-stat-store-gnus-article-buffer))
660
661(when spam-stat-install-hooks
662 (spam-stat-install-hooks-function))
663
664(defun spam-stat-unload-hook ()
01c52d31 665 "Uninstall the spam-stat function hooks."
23f87bed
MB
666 (interactive)
667 (remove-hook 'nnmail-prepare-incoming-message-hook
668 'spam-stat-store-current-buffer)
669 (remove-hook 'gnus-select-article-hook
670 'spam-stat-store-gnus-article-buffer))
671
e0281050
RS
672(add-hook 'spam-stat-unload-hook 'spam-stat-unload-hook)
673
23f87bed
MB
674(provide 'spam-stat)
675
cbee283d 676;; arch-tag: ff1d2200-8ddb-42fb-bb7b-1b5e20448554
23f87bed 677;;; spam-stat.el ends here