Commit | Line | Data |
---|---|---|
eec82323 | 1 | ;;; nnweb.el --- retrieving articles via web search engines |
e84b4b86 TTN |
2 | |
3 | ;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, | |
114f9c96 | 4 | ;; 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc. |
eec82323 | 5 | |
6748645f | 6 | ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org> |
eec82323 LMI |
7 | ;; Keywords: news |
8 | ||
9 | ;; This file is part of GNU Emacs. | |
10 | ||
5e809f55 | 11 | ;; GNU Emacs is free software: you can redistribute it and/or modify |
eec82323 | 12 | ;; it under the terms of the GNU General Public License as published by |
5e809f55 GM |
13 | ;; the Free Software Foundation, either version 3 of the License, or |
14 | ;; (at your option) any later version. | |
eec82323 LMI |
15 | |
16 | ;; GNU Emacs is distributed in the hope that it will be useful, | |
17 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
18 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
19 | ;; GNU General Public License for more details. | |
20 | ||
21 | ;; You should have received a copy of the GNU General Public License | |
5e809f55 | 22 | ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>. |
eec82323 LMI |
23 | |
24 | ;;; Commentary: | |
25 | ||
23f87bed | 26 | ;; Note: You need to have `w3' installed for some functions to work. |
eec82323 LMI |
27 | |
28 | ;;; Code: | |
29 | ||
5ab7173c RS |
30 | (eval-when-compile (require 'cl)) |
31 | ||
eec82323 LMI |
32 | (require 'nnoo) |
33 | (require 'message) | |
34 | (require 'gnus-util) | |
35 | (require 'gnus) | |
eec82323 | 36 | (require 'nnmail) |
16409b0b | 37 | (require 'mm-util) |
23f87bed MB |
38 | (require 'mm-url) |
39 | (eval-and-compile | |
0d5dc4a5 | 40 | (ignore-errors |
23f87bed MB |
41 | (require 'url))) |
42 | (autoload 'w3-parse-buffer "w3-parse") | |
eec82323 LMI |
43 | |
44 | (nnoo-declare nnweb) | |
45 | ||
46 | (defvoo nnweb-directory (nnheader-concat gnus-directory "nnweb/") | |
47 | "Where nnweb will save its files.") | |
48 | ||
95fa1ff7 | 49 | (defvoo nnweb-type 'google |
6748645f | 50 | "What search engine type is being used. |
23f87bed | 51 | Valid types include `google', `dejanews', and `gmane'.") |
eec82323 | 52 | |
16409b0b | 53 | (defvar nnweb-type-definition |
23f87bed | 54 | |
4a2358e9 | 55 | (id . "http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source") |
46e8fe3d | 56 | (result . "http://groups.google.com/group/%s/msg/%s?dmode=source") |
4a2358e9 | 57 | (article . nnweb-google-wash-article) |
95fa1ff7 SZ |
58 | (reference . identity) |
59 | (map . nnweb-google-create-mapping) | |
60 | (search . nnweb-google-search) | |
4a2358e9 MB |
61 | (address . "http://groups.google.com/groups") |
62 | (base . "http://groups.google.com") | |
95fa1ff7 SZ |
63 | (identifier . nnweb-google-identity)) |
64 | (dejanews ;; alias of google | |
46e8fe3d MB |
65 | (id . "http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source") |
66 | (result . "http://groups.google.com/group/%s/msg/%s?dmode=source") | |
67 | (article . nnweb-google-wash-article) | |
95fa1ff7 SZ |
68 | (reference . identity) |
69 | (map . nnweb-google-create-mapping) | |
70 | (search . nnweb-google-search) | |
71 | (address . "http://groups.google.com/groups") | |
5f5475ac | 72 | (base . "http://groups.google.com") |
95fa1ff7 | 73 | (identifier . nnweb-google-identity)) |
23f87bed MB |
74 | (gmane |
75 | (article . nnweb-gmane-wash-article) | |
76 | (id . "http://gmane.org/view.php?group=%s") | |
77 | (reference . identity) | |
78 | (map . nnweb-gmane-create-mapping) | |
79 | (search . nnweb-gmane-search) | |
719120ef | 80 | (address . "http://search.gmane.org/nov.php") |
23f87bed | 81 | (identifier . nnweb-gmane-identity))) |
eec82323 LMI |
82 | "Type-definition alist.") |
83 | ||
84 | (defvoo nnweb-search nil | |
23f87bed | 85 | "Search string to feed to Google.") |
eec82323 | 86 | |
6748645f | 87 | (defvoo nnweb-max-hits 999 |
eec82323 LMI |
88 | "Maximum number of hits to display.") |
89 | ||
90 | (defvoo nnweb-ephemeral-p nil | |
91 | "Whether this nnweb server is ephemeral.") | |
92 | ||
93 | ;;; Internal variables | |
94 | ||
95 | (defvoo nnweb-articles nil) | |
96 | (defvoo nnweb-buffer nil) | |
719120ef | 97 | (defvoo nnweb-group-alist nil) |
eec82323 LMI |
98 | (defvoo nnweb-group nil) |
99 | (defvoo nnweb-hashtb nil) | |
100 | ||
101 | ;;; Interface functions | |
102 | ||
103 | (nnoo-define-basics nnweb) | |
104 | ||
105 | (deffoo nnweb-retrieve-headers (articles &optional group server fetch-old) | |
106 | (nnweb-possibly-change-server group server) | |
20a673b2 | 107 | (with-current-buffer nntp-server-buffer |
eec82323 LMI |
108 | (erase-buffer) |
109 | (let (article header) | |
16409b0b GM |
110 | (mm-with-unibyte-current-buffer |
111 | (while (setq article (pop articles)) | |
112 | (when (setq header (cadr (assq article nnweb-articles))) | |
113 | (nnheader-insert-nov header)))) | |
eec82323 LMI |
114 | 'nov))) |
115 | ||
116 | (deffoo nnweb-request-scan (&optional group server) | |
117 | (nnweb-possibly-change-server group server) | |
95fa1ff7 | 118 | (if nnweb-ephemeral-p |
46e8fe3d MB |
119 | (setq nnweb-hashtb (gnus-make-hashtable 4095)) |
120 | (unless nnweb-articles | |
121 | (nnweb-read-overview group))) | |
eec82323 LMI |
122 | (funcall (nnweb-definition 'map)) |
123 | (unless nnweb-ephemeral-p | |
124 | (nnweb-write-active) | |
125 | (nnweb-write-overview group))) | |
126 | ||
127 | (deffoo nnweb-request-group (group &optional server dont-check) | |
46e8fe3d MB |
128 | (nnweb-possibly-change-server group server) |
129 | (unless (or nnweb-ephemeral-p | |
6203370b MB |
130 | dont-check |
131 | nnweb-articles) | |
46e8fe3d | 132 | (nnweb-read-overview group)) |
eec82323 LMI |
133 | (cond |
134 | ((not nnweb-articles) | |
135 | (nnheader-report 'nnweb "No matching articles")) | |
136 | (t | |
137 | (let ((active (if nnweb-ephemeral-p | |
138 | (cons (caar nnweb-articles) | |
139 | (caar (last nnweb-articles))) | |
140 | (cadr (assoc group nnweb-group-alist))))) | |
141 | (nnheader-report 'nnweb "Opened group %s" group) | |
142 | (nnheader-insert | |
143 | "211 %d %d %d %s\n" (length nnweb-articles) | |
144 | (car active) (cdr active) group))))) | |
145 | ||
146 | (deffoo nnweb-close-group (group &optional server) | |
147 | (nnweb-possibly-change-server group server) | |
148 | (when (gnus-buffer-live-p nnweb-buffer) | |
20a673b2 | 149 | (with-current-buffer nnweb-buffer |
eec82323 LMI |
150 | (set-buffer-modified-p nil) |
151 | (kill-buffer nnweb-buffer))) | |
152 | t) | |
153 | ||
154 | (deffoo nnweb-request-article (article &optional group server buffer) | |
155 | (nnweb-possibly-change-server group server) | |
20a673b2 | 156 | (with-current-buffer (or buffer nntp-server-buffer) |
eec82323 LMI |
157 | (let* ((header (cadr (assq article nnweb-articles))) |
158 | (url (and header (mail-header-xref header)))) | |
159 | (when (or (and url | |
16409b0b | 160 | (mm-with-unibyte-current-buffer |
23f87bed | 161 | (mm-url-insert url))) |
eec82323 LMI |
162 | (and (stringp article) |
163 | (nnweb-definition 'id t) | |
164 | (let ((fetch (nnweb-definition 'id)) | |
95fa1ff7 | 165 | art active) |
eec82323 LMI |
166 | (when (string-match "^<\\(.*\\)>$" article) |
167 | (setq art (match-string 1 article))) | |
95fa1ff7 | 168 | (when (and fetch art) |
7ce31649 MB |
169 | (setq url (format fetch |
170 | (mm-url-form-encode-xwfu art))) | |
95fa1ff7 | 171 | (mm-with-unibyte-current-buffer |
23f87bed | 172 | (mm-url-insert url)) |
95fa1ff7 SZ |
173 | (if (nnweb-definition 'reference t) |
174 | (setq article | |
175 | (funcall (nnweb-definition | |
176 | 'reference) article))))))) | |
eec82323 | 177 | (unless nnheader-callback-function |
95fa1ff7 | 178 | (funcall (nnweb-definition 'article))) |
eec82323 | 179 | (nnheader-report 'nnweb "Fetched article %s" article) |
16409b0b | 180 | (cons group (and (numberp article) article)))))) |
eec82323 LMI |
181 | |
182 | (deffoo nnweb-close-server (&optional server) | |
183 | (when (and (nnweb-server-opened server) | |
184 | (gnus-buffer-live-p nnweb-buffer)) | |
20a673b2 | 185 | (with-current-buffer nnweb-buffer |
eec82323 LMI |
186 | (set-buffer-modified-p nil) |
187 | (kill-buffer nnweb-buffer))) | |
188 | (nnoo-close-server 'nnweb server)) | |
189 | ||
190 | (deffoo nnweb-request-list (&optional server) | |
191 | (nnweb-possibly-change-server nil server) | |
20a673b2 | 192 | (with-current-buffer nntp-server-buffer |
46e8fe3d | 193 | (nnmail-generate-active (list (assoc server nnweb-group-alist))) |
eec82323 LMI |
194 | t)) |
195 | ||
196 | (deffoo nnweb-request-update-info (group info &optional server) | |
16409b0b | 197 | (nnweb-possibly-change-server group server)) |
eec82323 LMI |
198 | |
199 | (deffoo nnweb-asynchronous-p () | |
23f87bed | 200 | nil) |
eec82323 LMI |
201 | |
202 | (deffoo nnweb-request-create-group (group &optional server args) | |
203 | (nnweb-possibly-change-server nil server) | |
204 | (nnweb-request-delete-group group) | |
46e8fe3d | 205 | (push `(,group ,(cons 1 0)) nnweb-group-alist) |
eec82323 LMI |
206 | (nnweb-write-active) |
207 | t) | |
208 | ||
209 | (deffoo nnweb-request-delete-group (group &optional force server) | |
210 | (nnweb-possibly-change-server group server) | |
16409b0b GM |
211 | (gnus-pull group nnweb-group-alist t) |
212 | (nnweb-write-active) | |
eec82323 LMI |
213 | (gnus-delete-file (nnweb-overview-file group)) |
214 | t) | |
215 | ||
216 | (nnoo-define-skeleton nnweb) | |
217 | ||
218 | ;;; Internal functions | |
219 | ||
220 | (defun nnweb-read-overview (group) | |
221 | "Read the overview of GROUP and build the map." | |
222 | (when (file-exists-p (nnweb-overview-file group)) | |
16409b0b | 223 | (mm-with-unibyte-buffer |
eec82323 LMI |
224 | (nnheader-insert-file-contents (nnweb-overview-file group)) |
225 | (goto-char (point-min)) | |
226 | (let (header) | |
227 | (while (not (eobp)) | |
228 | (setq header (nnheader-parse-nov)) | |
229 | (forward-line 1) | |
230 | (push (list (mail-header-number header) | |
231 | header (mail-header-xref header)) | |
232 | nnweb-articles) | |
233 | (nnweb-set-hashtb header (car nnweb-articles))))))) | |
234 | ||
235 | (defun nnweb-write-overview (group) | |
236 | "Write the overview file for GROUP." | |
16409b0b | 237 | (with-temp-file (nnweb-overview-file group) |
eec82323 LMI |
238 | (let ((articles nnweb-articles)) |
239 | (while articles | |
240 | (nnheader-insert-nov (cadr (pop articles))))))) | |
241 | ||
242 | (defun nnweb-set-hashtb (header data) | |
243 | (gnus-sethash (nnweb-identifier (mail-header-xref header)) | |
244 | data nnweb-hashtb)) | |
245 | ||
246 | (defun nnweb-get-hashtb (url) | |
247 | (gnus-gethash (nnweb-identifier url) nnweb-hashtb)) | |
248 | ||
249 | (defun nnweb-identifier (ident) | |
250 | (funcall (nnweb-definition 'identifier) ident)) | |
251 | ||
252 | (defun nnweb-overview-file (group) | |
253 | "Return the name of the overview file of GROUP." | |
254 | (nnheader-concat nnweb-directory group ".overview")) | |
255 | ||
256 | (defun nnweb-write-active () | |
257 | "Save the active file." | |
16409b0b GM |
258 | (gnus-make-directory nnweb-directory) |
259 | (with-temp-file (nnheader-concat nnweb-directory "active") | |
eec82323 LMI |
260 | (prin1 `(setq nnweb-group-alist ',nnweb-group-alist) (current-buffer)))) |
261 | ||
262 | (defun nnweb-read-active () | |
263 | "Read the active file." | |
264 | (load (nnheader-concat nnweb-directory "active") t t t)) | |
265 | ||
266 | (defun nnweb-definition (type &optional noerror) | |
267 | "Return the definition of TYPE." | |
268 | (let ((def (cdr (assq type (assq nnweb-type nnweb-type-definition))))) | |
269 | (when (and (not def) | |
270 | (not noerror)) | |
271 | (error "Undefined definition %s" type)) | |
272 | def)) | |
273 | ||
274 | (defun nnweb-possibly-change-server (&optional group server) | |
eec82323 LMI |
275 | (when server |
276 | (unless (nnweb-server-opened server) | |
46e8fe3d MB |
277 | (nnweb-open-server server)) |
278 | (nnweb-init server)) | |
eec82323 LMI |
279 | (unless nnweb-group-alist |
280 | (nnweb-read-active)) | |
95fa1ff7 SZ |
281 | (unless nnweb-hashtb |
282 | (setq nnweb-hashtb (gnus-make-hashtable 4095))) | |
eec82323 | 283 | (when group |
46e8fe3d | 284 | (setq nnweb-group group))) |
eec82323 LMI |
285 | |
286 | (defun nnweb-init (server) | |
287 | "Initialize buffers and such." | |
288 | (unless (gnus-buffer-live-p nnweb-buffer) | |
289 | (setq nnweb-buffer | |
3b728e95 SM |
290 | (save-current-buffer |
291 | (nnheader-set-temp-buffer | |
292 | (format " *nnweb %s %s %s*" | |
293 | nnweb-type nnweb-search server)) | |
294 | (mm-disable-multibyte) | |
295 | (current-buffer))))) | |
eec82323 | 296 | |
95fa1ff7 | 297 | ;;; |
4a2358e9 | 298 | ;;; groups.google.com |
95fa1ff7 SZ |
299 | ;;; |
300 | ||
301 | (defun nnweb-google-wash-article () | |
4a2358e9 | 302 | ;; We have Google's masked e-mail addresses here. :-/ |
719120ef | 303 | (let ((case-fold-search t) |
0565caeb MB |
304 | (start-re "<pre>[\r\n ]*") |
305 | (end-re "[\r\n ]*</pre>")) | |
95fa1ff7 | 306 | (goto-char (point-min)) |
d752cf53 MB |
307 | (if (save-excursion |
308 | (or (re-search-forward "The requested message.*could not be found." | |
309 | nil t) | |
719120ef MB |
310 | (not (and (re-search-forward start-re nil t) |
311 | (re-search-forward end-re nil t))))) | |
d752cf53 MB |
312 | ;; FIXME: Don't know how to indicate "not found". |
313 | ;; Should this function throw an error? --rsteib | |
314 | (progn | |
315 | (gnus-message 3 "Requested article not found") | |
316 | (erase-buffer)) | |
317 | (delete-region (point-min) | |
719120ef | 318 | (re-search-forward start-re)) |
d752cf53 | 319 | (goto-char (point-min)) |
719120ef MB |
320 | (delete-region (progn |
321 | (re-search-forward end-re) | |
322 | (match-beginning 0)) | |
d752cf53 MB |
323 | (point-max)) |
324 | (mm-url-decode-entities)))) | |
95fa1ff7 SZ |
325 | |
326 | (defun nnweb-google-parse-1 (&optional Message-ID) | |
46e8fe3d | 327 | "Parse search result in current buffer." |
95fa1ff7 SZ |
328 | (let ((i 0) |
329 | (case-fold-search t) | |
330 | (active (cadr (assoc nnweb-group nnweb-group-alist))) | |
331 | Subject Score Date Newsgroups From | |
332 | map url mid) | |
333 | (unless active | |
46e8fe3d | 334 | (push (list nnweb-group (setq active (cons 1 0))) |
95fa1ff7 SZ |
335 | nnweb-group-alist)) |
336 | ;; Go through all the article hits on this page. | |
337 | (goto-char (point-min)) | |
46e8fe3d MB |
338 | (while |
339 | (re-search-forward | |
340 | "a +href=\"/group/\\([^>\"]+\\)/browse_thread/[^>]+#\\([0-9a-f]+\\)" | |
341 | nil t) | |
342 | (setq Newsgroups (match-string-no-properties 1) | |
343 | ;; Note: Starting with Google Groups 2, `mid' is a Google-internal | |
344 | ;; ID, not a proper Message-ID. | |
345 | mid (match-string-no-properties 2) | |
debad045 | 346 | url (format |
46e8fe3d | 347 | (nnweb-definition 'result) Newsgroups mid)) |
95fa1ff7 SZ |
348 | (narrow-to-region (search-forward ">" nil t) |
349 | (search-forward "</a>" nil t)) | |
23f87bed MB |
350 | (mm-url-remove-markup) |
351 | (mm-url-decode-entities) | |
95fa1ff7 SZ |
352 | (setq Subject (buffer-string)) |
353 | (goto-char (point-max)) | |
354 | (widen) | |
46e8fe3d | 355 | (narrow-to-region (point) |
c91f4b83 | 356 | (search-forward "</table" nil t)) |
46e8fe3d MB |
357 | |
358 | (mm-url-remove-markup) | |
359 | (mm-url-decode-entities) | |
c91f4b83 MB |
360 | (goto-char (point-max)) |
361 | (when | |
362 | (re-search-backward | |
14e8de0c | 363 | "^\\(?:\\(\\w+\\) \\([0-9]+\\)\\|\\S-+\\)\\(?: \\([0-9]\\{4\\}\\)\\)? by ?\\(.*\\)" |
c91f4b83 | 364 | nil t) |
aa260d63 MB |
365 | (setq Date (if (match-string 1) |
366 | (format "%s %s 00:00:00 %s" | |
367 | (match-string 1) | |
368 | (match-string 2) | |
369 | (or (match-string 3) | |
370 | (substring (current-time-string) -4))) | |
371 | (current-time-string))) | |
c91f4b83 | 372 | (setq From (match-string 4))) |
46e8fe3d | 373 | (widen) |
95fa1ff7 SZ |
374 | (incf i) |
375 | (unless (nnweb-get-hashtb url) | |
376 | (push | |
377 | (list | |
378 | (incf (cdr active)) | |
379 | (make-full-mail-header | |
380 | (cdr active) (if Newsgroups | |
381 | (concat "(" Newsgroups ") " Subject) | |
382 | Subject) | |
383 | From Date (or Message-ID mid) | |
384 | nil 0 0 url)) | |
385 | map) | |
386 | (nnweb-set-hashtb (cadar map) (car map)))) | |
387 | map)) | |
388 | ||
389 | (defun nnweb-google-reference (id) | |
390 | (let ((map (nnweb-google-parse-1 id)) header) | |
391 | (setq nnweb-articles | |
392 | (nconc nnweb-articles map)) | |
393 | (when (setq header (cadar map)) | |
394 | (mm-with-unibyte-current-buffer | |
23f87bed | 395 | (mm-url-insert (mail-header-xref header))) |
95fa1ff7 SZ |
396 | (caar map)))) |
397 | ||
398 | (defun nnweb-google-create-mapping () | |
debad045 | 399 | "Perform the search and create a number-to-url alist." |
20a673b2 | 400 | (with-current-buffer nnweb-buffer |
95fa1ff7 | 401 | (erase-buffer) |
719120ef | 402 | (nnheader-message 7 "Searching google...") |
95fa1ff7 | 403 | (when (funcall (nnweb-definition 'search) nnweb-search) |
23f87bed MB |
404 | (let ((more t) |
405 | (i 0)) | |
95fa1ff7 SZ |
406 | (while more |
407 | (setq nnweb-articles | |
408 | (nconc nnweb-articles (nnweb-google-parse-1))) | |
23f87bed MB |
409 | ;; Check if there are more articles to fetch |
410 | (goto-char (point-min)) | |
411 | (incf i 100) | |
412 | (if (or (not (re-search-forward | |
bd876f90 | 413 | "<a [^>]+href=\"\n?\\([^>\" \n\t]+\\)[^<]*<img[^>]+src=[^>]+next" |
719120ef | 414 | nil t)) |
23f87bed MB |
415 | (>= i nnweb-max-hits)) |
416 | (setq more nil) | |
417 | ;; Yup, there are more articles | |
5f5475ac | 418 | (setq more (concat (nnweb-definition 'base) (match-string 1))) |
23f87bed MB |
419 | (when more |
420 | (erase-buffer) | |
719120ef | 421 | (nnheader-message 7 "Searching google...(%d)" i) |
23f87bed | 422 | (mm-url-insert more)))) |
95fa1ff7 | 423 | ;; Return the articles in the right order. |
719120ef | 424 | (nnheader-message 7 "Searching google...done") |
95fa1ff7 SZ |
425 | (setq nnweb-articles |
426 | (sort nnweb-articles 'car-less-than-car)))))) | |
427 | ||
428 | (defun nnweb-google-search (search) | |
23f87bed | 429 | (mm-url-insert |
95fa1ff7 SZ |
430 | (concat |
431 | (nnweb-definition 'address) | |
432 | "?" | |
23f87bed | 433 | (mm-url-encode-www-form-urlencoded |
95fa1ff7 | 434 | `(("q" . ,search) |
7ce31649 MB |
435 | ("num" . ,(number-to-string |
436 | (min 100 nnweb-max-hits))) | |
95fa1ff7 | 437 | ("hq" . "") |
5f5475ac | 438 | ("hl" . "en") |
95fa1ff7 SZ |
439 | ("lr" . "") |
440 | ("safe" . "off") | |
46e8fe3d MB |
441 | ("sites" . "groups") |
442 | ("filter" . "0"))))) | |
95fa1ff7 SZ |
443 | t) |
444 | ||
445 | (defun nnweb-google-identity (url) | |
446 | "Return an unique identifier based on URL." | |
447 | (if (string-match "selm=\\([^ &>]+\\)" url) | |
448 | (match-string 1 url) | |
449 | url)) | |
450 | ||
23f87bed MB |
451 | ;;; |
452 | ;;; gmane.org | |
453 | ;;; | |
454 | (defun nnweb-gmane-create-mapping () | |
455 | "Perform the search and create a number-to-url alist." | |
20a673b2 | 456 | (with-current-buffer nnweb-buffer |
719120ef MB |
457 | (let ((case-fold-search t) |
458 | (active (or (cadr (assoc nnweb-group nnweb-group-alist)) | |
459 | (cons 1 0))) | |
460 | map) | |
461 | (erase-buffer) | |
462 | (nnheader-message 7 "Searching Gmane..." ) | |
463 | (when (funcall (nnweb-definition 'search) nnweb-search) | |
23f87bed | 464 | (goto-char (point-min)) |
719120ef MB |
465 | ;; Skip the status line |
466 | (forward-line 1) | |
467 | ;; Thanks to Olly Betts we now have NOV lines in our buffer! | |
468 | (while (not (eobp)) | |
469 | (unless (or (eolp) (looking-at "\x0d")) | |
470 | (let ((header (nnheader-parse-nov))) | |
471 | (let ((xref (mail-header-xref header)) | |
472 | (from (mail-header-from header)) | |
473 | (subject (mail-header-subject header)) | |
474 | (rfc2047-encoding-type 'mime)) | |
ba361211 | 475 | (when (string-match " \\([^:]+\\)[:/]\\([0-9]+\\)" xref) |
719120ef MB |
476 | (mail-header-set-xref |
477 | header | |
478 | (format "http://article.gmane.org/%s/%s/raw" | |
479 | (match-string 1 xref) | |
480 | (match-string 2 xref)))) | |
481 | ||
482 | ;; Add host part to gmane-encrypted addresses | |
483 | (when (string-match "@$" from) | |
484 | (mail-header-set-from header | |
485 | (concat from "public.gmane.org"))) | |
486 | ||
487 | (mail-header-set-subject header | |
488 | (rfc2047-encode-string subject)) | |
489 | ||
490 | (unless (nnweb-get-hashtb (mail-header-xref header)) | |
ba361211 MB |
491 | (mail-header-set-number header (incf (cdr active))) |
492 | (push (list (mail-header-number header) header) map) | |
719120ef MB |
493 | (nnweb-set-hashtb (cadar map) (car map)))))) |
494 | (forward-line 1))) | |
495 | (nnheader-message 7 "Searching Gmane...done") | |
496 | (setq nnweb-articles | |
497 | (sort (nconc nnweb-articles map) 'car-less-than-car))))) | |
23f87bed MB |
498 | |
499 | (defun nnweb-gmane-wash-article () | |
500 | (let ((case-fold-search t)) | |
501 | (goto-char (point-min)) | |
719120ef MB |
502 | (when (search-forward "<!--X-Head-of-Message-->" nil t) |
503 | (delete-region (point-min) (point)) | |
504 | (goto-char (point-min)) | |
505 | (while (looking-at "^<li><em>\\([^ ]+\\)</em>.*</li>") | |
506 | (replace-match "\\1\\2" t) | |
507 | (forward-line 1)) | |
508 | (mm-url-remove-markup)))) | |
23f87bed MB |
509 | |
510 | (defun nnweb-gmane-search (search) | |
511 | (mm-url-insert | |
512 | (concat | |
513 | (nnweb-definition 'address) | |
514 | "?" | |
515 | (mm-url-encode-www-form-urlencoded | |
719120ef | 516 | `(("query" . ,search) |
01c52d31 MB |
517 | ("HITSPERPAGE" . ,(number-to-string nnweb-max-hits)) |
518 | ;;("TOPDOC" . "1000") | |
519 | )))) | |
23f87bed | 520 | (setq buffer-file-name nil) |
765d4319 | 521 | (unless (featurep 'xemacs) (set-buffer-multibyte t)) |
719120ef | 522 | (mm-decode-coding-region (point-min) (point-max) 'utf-8) |
23f87bed MB |
523 | t) |
524 | ||
23f87bed MB |
525 | (defun nnweb-gmane-identity (url) |
526 | "Return a unique identifier based on URL." | |
527 | (if (string-match "group=\\(.+\\)" url) | |
528 | (match-string 1 url) | |
529 | url)) | |
530 | ||
16409b0b GM |
531 | ;;; |
532 | ;;; General web/w3 interface utility functions | |
533 | ;;; | |
534 | ||
535 | (defun nnweb-insert-html (parse) | |
536 | "Insert HTML based on a w3 parse tree." | |
537 | (if (stringp parse) | |
944c87e0 SM |
538 | ;; We used to call nnheader-string-as-multibyte here, but it cannot |
539 | ;; be right, so I removed it. If a bug shows up because of this change, | |
540 | ;; please do not blindly revert the change, but help me find the real | |
541 | ;; cause of the bug instead. --Stef | |
542 | (insert parse) | |
16409b0b GM |
543 | (insert "<" (symbol-name (car parse)) " ") |
544 | (insert (mapconcat | |
545 | (lambda (param) | |
546 | (concat (symbol-name (car param)) "=" | |
547 | (prin1-to-string | |
548 | (if (consp (cdr param)) | |
549 | (cadr param) | |
550 | (cdr param))))) | |
551 | (nth 1 parse) | |
552 | " ")) | |
553 | (insert ">\n") | |
01c52d31 | 554 | (mapc 'nnweb-insert-html (nth 2 parse)) |
16409b0b GM |
555 | (insert "</" (symbol-name (car parse)) ">\n"))) |
556 | ||
16409b0b GM |
557 | (defun nnweb-parse-find (type parse &optional maxdepth) |
558 | "Find the element of TYPE in PARSE." | |
559 | (catch 'found | |
560 | (nnweb-parse-find-1 type parse maxdepth))) | |
561 | ||
562 | (defun nnweb-parse-find-1 (type contents maxdepth) | |
563 | (when (or (null maxdepth) | |
564 | (not (zerop maxdepth))) | |
565 | (when (consp contents) | |
566 | (when (eq (car contents) type) | |
567 | (throw 'found contents)) | |
568 | (when (listp (cdr contents)) | |
569 | (dolist (element contents) | |
570 | (when (consp element) | |
571 | (nnweb-parse-find-1 type element | |
572 | (and maxdepth (1- maxdepth))))))))) | |
573 | ||
574 | (defun nnweb-parse-find-all (type parse) | |
575 | "Find all elements of TYPE in PARSE." | |
576 | (catch 'found | |
577 | (nnweb-parse-find-all-1 type parse))) | |
578 | ||
579 | (defun nnweb-parse-find-all-1 (type contents) | |
580 | (let (result) | |
581 | (when (consp contents) | |
582 | (if (eq (car contents) type) | |
583 | (push contents result) | |
584 | (when (listp (cdr contents)) | |
585 | (dolist (element contents) | |
586 | (when (consp element) | |
587 | (setq result | |
588 | (nconc result (nnweb-parse-find-all-1 type element)))))))) | |
589 | result)) | |
590 | ||
591 | (defvar nnweb-text) | |
592 | (defun nnweb-text (parse) | |
593 | "Return a list of text contents in PARSE." | |
594 | (let ((nnweb-text nil)) | |
595 | (nnweb-text-1 parse) | |
596 | (nreverse nnweb-text))) | |
597 | ||
598 | (defun nnweb-text-1 (contents) | |
599 | (dolist (element contents) | |
600 | (if (stringp element) | |
601 | (push element nnweb-text) | |
602 | (when (and (consp element) | |
603 | (listp (cdr element))) | |
604 | (nnweb-text-1 element))))) | |
605 | ||
eec82323 LMI |
606 | (provide 'nnweb) |
607 | ||
608 | ;;; nnweb.el ends here |