1 ;;; nnweb.el --- retrieving articles via web search engines
3 ;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
4 ;; 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
6 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
9 ;; This file is part of GNU Emacs.
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
26 ;; Note: You need to have `w3' installed for some functions to work.
30 (eval-when-compile (require 'cl
))
42 (autoload 'w3-parse-buffer
"w3-parse")
46 (defvoo nnweb-directory
(nnheader-concat gnus-directory
"nnweb/")
47 "Where nnweb will save its files.")
49 (defvoo nnweb-type
'google
50 "What search engine type is being used.
51 Valid types include `google', `dejanews', and `gmane'.")
53 (defvar nnweb-type-definition
55 (id .
"http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
56 (result .
"http://groups.google.com/group/%s/msg/%s?dmode=source")
57 (article . nnweb-google-wash-article
)
58 (reference . identity
)
59 (map . nnweb-google-create-mapping
)
60 (search . nnweb-google-search
)
61 (address .
"http://groups.google.com/groups")
62 (base .
"http://groups.google.com")
63 (identifier . nnweb-google-identity
))
64 (dejanews ;; alias of google
65 (id .
"http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
66 (result .
"http://groups.google.com/group/%s/msg/%s?dmode=source")
67 (article . nnweb-google-wash-article
)
68 (reference . identity
)
69 (map . nnweb-google-create-mapping
)
70 (search . nnweb-google-search
)
71 (address .
"http://groups.google.com/groups")
72 (base .
"http://groups.google.com")
73 (identifier . nnweb-google-identity
))
75 (article . nnweb-gmane-wash-article
)
76 (id .
"http://gmane.org/view.php?group=%s")
77 (reference . identity
)
78 (map . nnweb-gmane-create-mapping
)
79 (search . nnweb-gmane-search
)
80 (address .
"http://search.gmane.org/nov.php")
81 (identifier . nnweb-gmane-identity
)))
82 "Type-definition alist.")
84 (defvoo nnweb-search nil
85 "Search string to feed to Google.")
87 (defvoo nnweb-max-hits
999
88 "Maximum number of hits to display.")
90 (defvoo nnweb-ephemeral-p nil
91 "Whether this nnweb server is ephemeral.")
93 ;;; Internal variables
95 (defvoo nnweb-articles nil
)
96 (defvoo nnweb-buffer nil
)
97 (defvoo nnweb-group-alist nil
)
98 (defvoo nnweb-group nil
)
99 (defvoo nnweb-hashtb nil
)
101 ;;; Interface functions
103 (nnoo-define-basics nnweb
)
105 (deffoo nnweb-retrieve-headers
(articles &optional group server fetch-old
)
106 (nnweb-possibly-change-server group server
)
107 (with-current-buffer nntp-server-buffer
109 (let (article header
)
110 (mm-with-unibyte-current-buffer
111 (while (setq article
(pop articles
))
112 (when (setq header
(cadr (assq article nnweb-articles
)))
113 (nnheader-insert-nov header
))))
116 (deffoo nnweb-request-scan
(&optional group server
)
117 (nnweb-possibly-change-server group server
)
118 (if nnweb-ephemeral-p
119 (setq nnweb-hashtb
(gnus-make-hashtable 4095))
120 (unless nnweb-articles
121 (nnweb-read-overview group
)))
122 (funcall (nnweb-definition 'map
))
123 (unless nnweb-ephemeral-p
125 (nnweb-write-overview group
)))
127 (deffoo nnweb-request-group
(group &optional server dont-check
)
128 (nnweb-possibly-change-server group server
)
129 (unless (or nnweb-ephemeral-p
132 (nnweb-read-overview group
))
134 ((not nnweb-articles
)
135 (nnheader-report 'nnweb
"No matching articles"))
137 (let ((active (if nnweb-ephemeral-p
138 (cons (caar nnweb-articles
)
139 (caar (last nnweb-articles
)))
140 (cadr (assoc group nnweb-group-alist
)))))
141 (nnheader-report 'nnweb
"Opened group %s" group
)
143 "211 %d %d %d %s\n" (length nnweb-articles
)
144 (car active
) (cdr active
) group
)))))
146 (deffoo nnweb-close-group
(group &optional server
)
147 (nnweb-possibly-change-server group server
)
148 (when (gnus-buffer-live-p nnweb-buffer
)
149 (with-current-buffer nnweb-buffer
150 (set-buffer-modified-p nil
)
151 (kill-buffer nnweb-buffer
)))
154 (deffoo nnweb-request-article
(article &optional group server buffer
)
155 (nnweb-possibly-change-server group server
)
156 (with-current-buffer (or buffer nntp-server-buffer
)
157 (let* ((header (cadr (assq article nnweb-articles
)))
158 (url (and header
(mail-header-xref header
))))
160 (mm-with-unibyte-current-buffer
161 (mm-url-insert url
)))
162 (and (stringp article
)
163 (nnweb-definition 'id t
)
164 (let ((fetch (nnweb-definition 'id
))
166 (when (string-match "^<\\(.*\\)>$" article
)
167 (setq art
(match-string 1 article
)))
168 (when (and fetch art
)
169 (setq url
(format fetch
170 (mm-url-form-encode-xwfu art
)))
171 (mm-with-unibyte-current-buffer
173 (if (nnweb-definition 'reference t
)
175 (funcall (nnweb-definition
176 'reference
) article
)))))))
177 (unless nnheader-callback-function
178 (funcall (nnweb-definition 'article
)))
179 (nnheader-report 'nnweb
"Fetched article %s" article
)
180 (cons group
(and (numberp article
) article
))))))
182 (deffoo nnweb-close-server
(&optional server
)
183 (when (and (nnweb-server-opened server
)
184 (gnus-buffer-live-p nnweb-buffer
))
185 (with-current-buffer nnweb-buffer
186 (set-buffer-modified-p nil
)
187 (kill-buffer nnweb-buffer
)))
188 (nnoo-close-server 'nnweb server
))
190 (deffoo nnweb-request-list
(&optional server
)
191 (nnweb-possibly-change-server nil server
)
192 (with-current-buffer nntp-server-buffer
193 (nnmail-generate-active (list (assoc server nnweb-group-alist
)))
196 (deffoo nnweb-request-update-info
(group info
&optional server
)
197 (nnweb-possibly-change-server group server
))
199 (deffoo nnweb-asynchronous-p
()
202 (deffoo nnweb-request-create-group
(group &optional server args
)
203 (nnweb-possibly-change-server nil server
)
204 (nnweb-request-delete-group group
)
205 (push `(,group
,(cons 1 0)) nnweb-group-alist
)
209 (deffoo nnweb-request-delete-group
(group &optional force server
)
210 (nnweb-possibly-change-server group server
)
211 (gnus-pull group nnweb-group-alist t
)
213 (gnus-delete-file (nnweb-overview-file group
))
216 (nnoo-define-skeleton nnweb
)
218 ;;; Internal functions
220 (defun nnweb-read-overview (group)
221 "Read the overview of GROUP and build the map."
222 (when (file-exists-p (nnweb-overview-file group
))
223 (mm-with-unibyte-buffer
224 (nnheader-insert-file-contents (nnweb-overview-file group
))
225 (goto-char (point-min))
228 (setq header
(nnheader-parse-nov))
230 (push (list (mail-header-number header
)
231 header
(mail-header-xref header
))
233 (nnweb-set-hashtb header
(car nnweb-articles
)))))))
235 (defun nnweb-write-overview (group)
236 "Write the overview file for GROUP."
237 (with-temp-file (nnweb-overview-file group
)
238 (let ((articles nnweb-articles
))
240 (nnheader-insert-nov (cadr (pop articles
)))))))
242 (defun nnweb-set-hashtb (header data
)
243 (gnus-sethash (nnweb-identifier (mail-header-xref header
))
246 (defun nnweb-get-hashtb (url)
247 (gnus-gethash (nnweb-identifier url
) nnweb-hashtb
))
249 (defun nnweb-identifier (ident)
250 (funcall (nnweb-definition 'identifier
) ident
))
252 (defun nnweb-overview-file (group)
253 "Return the name of the overview file of GROUP."
254 (nnheader-concat nnweb-directory group
".overview"))
256 (defun nnweb-write-active ()
257 "Save the active file."
258 (gnus-make-directory nnweb-directory
)
259 (with-temp-file (nnheader-concat nnweb-directory
"active")
260 (prin1 `(setq nnweb-group-alist
',nnweb-group-alist
) (current-buffer))))
262 (defun nnweb-read-active ()
263 "Read the active file."
264 (load (nnheader-concat nnweb-directory
"active") t t t
))
266 (defun nnweb-definition (type &optional noerror
)
267 "Return the definition of TYPE."
268 (let ((def (cdr (assq type
(assq nnweb-type nnweb-type-definition
)))))
271 (error "Undefined definition %s" type
))
274 (defun nnweb-possibly-change-server (&optional group server
)
276 (unless (nnweb-server-opened server
)
277 (nnweb-open-server server
))
279 (unless nnweb-group-alist
282 (setq nnweb-hashtb
(gnus-make-hashtable 4095)))
284 (setq nnweb-group group
)))
286 (defun nnweb-init (server)
287 "Initialize buffers and such."
288 (unless (gnus-buffer-live-p nnweb-buffer
)
291 (nnheader-set-temp-buffer
292 (format " *nnweb %s %s %s*"
293 nnweb-type nnweb-search server
))
294 (mm-disable-multibyte)
298 ;;; groups.google.com
301 (defun nnweb-google-wash-article ()
302 ;; We have Google's masked e-mail addresses here. :-/
303 (let ((case-fold-search t
)
304 (start-re "<pre>[\r\n ]*")
305 (end-re "[\r\n ]*</pre>"))
306 (goto-char (point-min))
308 (or (re-search-forward "The requested message.*could not be found."
310 (not (and (re-search-forward start-re nil t
)
311 (re-search-forward end-re nil t
)))))
312 ;; FIXME: Don't know how to indicate "not found".
313 ;; Should this function throw an error? --rsteib
315 (gnus-message 3 "Requested article not found")
317 (delete-region (point-min)
318 (re-search-forward start-re
))
319 (goto-char (point-min))
320 (delete-region (progn
321 (re-search-forward end-re
)
324 (mm-url-decode-entities))))
326 (defun nnweb-google-parse-1 (&optional Message-ID
)
327 "Parse search result in current buffer."
330 (active (cadr (assoc nnweb-group nnweb-group-alist
)))
331 Subject Score Date Newsgroups From
334 (push (list nnweb-group
(setq active
(cons 1 0)))
336 ;; Go through all the article hits on this page.
337 (goto-char (point-min))
340 "a +href=\"/group/\\([^>\"]+\\)/browse_thread/[^>]+#\\([0-9a-f]+\\)"
342 (setq Newsgroups
(match-string-no-properties 1)
343 ;; Note: Starting with Google Groups 2, `mid' is a Google-internal
344 ;; ID, not a proper Message-ID.
345 mid
(match-string-no-properties 2)
347 (nnweb-definition 'result
) Newsgroups mid
))
348 (narrow-to-region (search-forward ">" nil t
)
349 (search-forward "</a>" nil t
))
350 (mm-url-remove-markup)
351 (mm-url-decode-entities)
352 (setq Subject
(buffer-string))
353 (goto-char (point-max))
355 (narrow-to-region (point)
356 (search-forward "</table" nil t
))
358 (mm-url-remove-markup)
359 (mm-url-decode-entities)
360 (goto-char (point-max))
363 "^\\(?:\\(\\w+\\) \\([0-9]+\\)\\|\\S-+\\)\\(?: \\([0-9]\\{4\\}\\)\\)? by ?\\(.*\\)"
365 (setq Date
(if (match-string 1)
366 (format "%s %s 00:00:00 %s"
370 (substring (current-time-string) -
4)))
371 (current-time-string)))
372 (setq From
(match-string 4)))
375 (unless (nnweb-get-hashtb url
)
379 (make-full-mail-header
380 (cdr active
) (if Newsgroups
381 (concat "(" Newsgroups
") " Subject
)
383 From Date
(or Message-ID mid
)
386 (nnweb-set-hashtb (cadar map
) (car map
))))
389 (defun nnweb-google-reference (id)
390 (let ((map (nnweb-google-parse-1 id
)) header
)
392 (nconc nnweb-articles map
))
393 (when (setq header
(cadar map
))
394 (mm-with-unibyte-current-buffer
395 (mm-url-insert (mail-header-xref header
)))
398 (defun nnweb-google-create-mapping ()
399 "Perform the search and create a number-to-url alist."
400 (with-current-buffer nnweb-buffer
402 (nnheader-message 7 "Searching google...")
403 (when (funcall (nnweb-definition 'search
) nnweb-search
)
408 (nconc nnweb-articles
(nnweb-google-parse-1)))
409 ;; Check if there are more articles to fetch
410 (goto-char (point-min))
412 (if (or (not (re-search-forward
413 "<a [^>]+href=\"\n?\\([^>\" \n\t]+\\)[^<]*<img[^>]+src=[^>]+next"
415 (>= i nnweb-max-hits
))
417 ;; Yup, there are more articles
418 (setq more
(concat (nnweb-definition 'base
) (match-string 1)))
421 (nnheader-message 7 "Searching google...(%d)" i
)
422 (mm-url-insert more
))))
423 ;; Return the articles in the right order.
424 (nnheader-message 7 "Searching google...done")
426 (sort nnweb-articles
'car-less-than-car
))))))
428 (defun nnweb-google-search (search)
431 (nnweb-definition 'address
)
433 (mm-url-encode-www-form-urlencoded
435 ("num" .
,(number-to-string
436 (min 100 nnweb-max-hits
)))
445 (defun nnweb-google-identity (url)
446 "Return an unique identifier based on URL."
447 (if (string-match "selm=\\([^ &>]+\\)" url
)
454 (defun nnweb-gmane-create-mapping ()
455 "Perform the search and create a number-to-url alist."
456 (with-current-buffer nnweb-buffer
457 (let ((case-fold-search t
)
458 (active (or (cadr (assoc nnweb-group nnweb-group-alist
))
462 (nnheader-message 7 "Searching Gmane..." )
463 (when (funcall (nnweb-definition 'search
) nnweb-search
)
464 (goto-char (point-min))
465 ;; Skip the status line
467 ;; Thanks to Olly Betts we now have NOV lines in our buffer!
469 (unless (or (eolp) (looking-at "\x0d"))
470 (let ((header (nnheader-parse-nov)))
471 (let ((xref (mail-header-xref header
))
472 (from (mail-header-from header
))
473 (subject (mail-header-subject header
))
474 (rfc2047-encoding-type 'mime
))
475 (when (string-match " \\([^:]+\\)[:/]\\([0-9]+\\)" xref
)
476 (mail-header-set-xref
478 (format "http://article.gmane.org/%s/%s/raw"
479 (match-string 1 xref
)
480 (match-string 2 xref
))))
482 ;; Add host part to gmane-encrypted addresses
483 (when (string-match "@$" from
)
484 (mail-header-set-from header
485 (concat from
"public.gmane.org")))
487 (mail-header-set-subject header
488 (rfc2047-encode-string subject
))
490 (unless (nnweb-get-hashtb (mail-header-xref header
))
491 (mail-header-set-number header
(incf (cdr active
)))
492 (push (list (mail-header-number header
) header
) map
)
493 (nnweb-set-hashtb (cadar map
) (car map
))))))
495 (nnheader-message 7 "Searching Gmane...done")
497 (sort (nconc nnweb-articles map
) 'car-less-than-car
)))))
499 (defun nnweb-gmane-wash-article ()
500 (let ((case-fold-search t
))
501 (goto-char (point-min))
502 (when (search-forward "<!--X-Head-of-Message-->" nil t
)
503 (delete-region (point-min) (point))
504 (goto-char (point-min))
505 (while (looking-at "^<li><em>\\([^ ]+\\)</em>.*</li>")
506 (replace-match "\\1\\2" t
)
508 (mm-url-remove-markup))))
510 (defun nnweb-gmane-search (search)
513 (nnweb-definition 'address
)
515 (mm-url-encode-www-form-urlencoded
516 `(("query" .
,search
)
517 ("HITSPERPAGE" .
,(number-to-string nnweb-max-hits
))
518 ;;("TOPDOC" . "1000")
520 (setq buffer-file-name nil
)
521 (unless (featurep 'xemacs
) (set-buffer-multibyte t
))
522 (mm-decode-coding-region (point-min) (point-max) 'utf-8
)
525 (defun nnweb-gmane-identity (url)
526 "Return a unique identifier based on URL."
527 (if (string-match "group=\\(.+\\)" url
)
532 ;;; General web/w3 interface utility functions
535 (defun nnweb-insert-html (parse)
536 "Insert HTML based on a w3 parse tree."
538 ;; We used to call nnheader-string-as-multibyte here, but it cannot
539 ;; be right, so I removed it. If a bug shows up because of this change,
540 ;; please do not blindly revert the change, but help me find the real
541 ;; cause of the bug instead. --Stef
543 (insert "<" (symbol-name (car parse
)) " ")
546 (concat (symbol-name (car param
)) "="
548 (if (consp (cdr param
))
554 (mapc 'nnweb-insert-html
(nth 2 parse
))
555 (insert "</" (symbol-name (car parse
)) ">\n")))
557 (defun nnweb-parse-find (type parse
&optional maxdepth
)
558 "Find the element of TYPE in PARSE."
560 (nnweb-parse-find-1 type parse maxdepth
)))
562 (defun nnweb-parse-find-1 (type contents maxdepth
)
563 (when (or (null maxdepth
)
564 (not (zerop maxdepth
)))
565 (when (consp contents
)
566 (when (eq (car contents
) type
)
567 (throw 'found contents
))
568 (when (listp (cdr contents
))
569 (dolist (element contents
)
570 (when (consp element
)
571 (nnweb-parse-find-1 type element
572 (and maxdepth
(1- maxdepth
)))))))))
574 (defun nnweb-parse-find-all (type parse
)
575 "Find all elements of TYPE in PARSE."
577 (nnweb-parse-find-all-1 type parse
)))
579 (defun nnweb-parse-find-all-1 (type contents
)
581 (when (consp contents
)
582 (if (eq (car contents
) type
)
583 (push contents result
)
584 (when (listp (cdr contents
))
585 (dolist (element contents
)
586 (when (consp element
)
588 (nconc result
(nnweb-parse-find-all-1 type element
))))))))
592 (defun nnweb-text (parse)
593 "Return a list of text contents in PARSE."
594 (let ((nnweb-text nil
))
596 (nreverse nnweb-text
)))
598 (defun nnweb-text-1 (contents)
599 (dolist (element contents
)
600 (if (stringp element
)
601 (push element nnweb-text
)
602 (when (and (consp element
)
603 (listp (cdr element
)))
604 (nnweb-text-1 element
)))))
608 ;;; nnweb.el ends here