HCoop / bpt / emacs.git / blame
| Commit | Line | Data |
|---|---|---|
| eec82323 | 1 | ;;; nnweb.el --- retrieving articles via web search engines |
| 16409b0b GM |
2 | ;; Copyright (C) 1996, 1997, 1998, 1999, 2000 |
| 3 | ;; Free Software Foundation, Inc. | |
| eec82323 | 4 | |
| 6748645f | 5 | ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org> |
| eec82323 LMI |
6 | ;; Keywords: news |
| 7 | ||
| 8 | ;; This file is part of GNU Emacs. | |
| 9 | ||
| 10 | ;; GNU Emacs is free software; you can redistribute it and/or modify | |
| 11 | ;; it under the terms of the GNU General Public License as published by | |
| 12 | ;; the Free Software Foundation; either version 2, or (at your option) | |
| 13 | ;; any later version. | |
| 14 | ||
| 15 | ;; GNU Emacs is distributed in the hope that it will be useful, | |
| 16 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 17 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 18 | ;; GNU General Public License for more details. | |
| 19 | ||
| 20 | ;; You should have received a copy of the GNU General Public License | |
| 21 | ;; along with GNU Emacs; see the file COPYING. If not, write to the | |
| 22 | ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
| 23 | ;; Boston, MA 02111-1307, USA. | |
| 24 | ||
| 25 | ;;; Commentary: | |
| 26 | ||
| 27 | ;; Note: You need to have `url' and `w3' installed for this | |
| 28 | ;; backend to work. | |
| 29 | ||
| 30 | ;;; Code: | |
| 31 | ||
| 5ab7173c RS |
32 | (eval-when-compile (require 'cl)) |
| 33 | ||
| eec82323 LMI |
34 | (require 'nnoo) |
| 35 | (require 'message) | |
| 36 | (require 'gnus-util) | |
| 37 | (require 'gnus) | |
| eec82323 | 38 | (require 'nnmail) |
| 16409b0b | 39 | (require 'mm-util) |
| 0d5dc4a5 DL |
40 | (eval-when-compile |
| 41 | (ignore-errors | |
| 16409b0b GM |
42 | (require 'w3) |
| 43 | (require 'url) | |
| 44 | (require 'w3-forms))) | |
| 45 | ||
| 0d5dc4a5 | 46 | ;; Report failure to find w3 at load time if appropriate. |
| 16409b0b GM |
47 | (unless noninteractive |
| 48 | (eval '(progn | |
| 49 | (require 'w3) | |
| 50 | (require 'url) | |
| 51 | (require 'w3-forms)))) | |
| eec82323 LMI |
52 | |
| 53 | (nnoo-declare nnweb) | |
| 54 | ||
| 55 | (defvoo nnweb-directory (nnheader-concat gnus-directory "nnweb/") | |
| 56 | "Where nnweb will save its files.") | |
| 57 | ||
| 58 | (defvoo nnweb-type 'dejanews | |
| 6748645f LMI |
59 | "What search engine type is being used. |
| 60 | Valid types include `dejanews', `dejanewsold', `reference', | |
| 61 | and `altavista'.") | |
| eec82323 | 62 | |
| 16409b0b | 63 | (defvar nnweb-type-definition |
| eec82323 | 64 | '((dejanews |
| 16409b0b GM |
65 | (article . ignore) |
| 66 | (id . "http://search.dejanews.com/msgid.xp?MID=%s&fmt=text") | |
| eec82323 LMI |
67 | (map . nnweb-dejanews-create-mapping) |
| 68 | (search . nnweb-dejanews-search) | |
| 16409b0b | 69 | (address . "http://www.deja.com/=dnc/qs.xp") |
| 6748645f LMI |
70 | (identifier . nnweb-dejanews-identity)) |
| 71 | (dejanewsold | |
| 16409b0b | 72 | (article . ignore) |
| 6748645f LMI |
73 | (map . nnweb-dejanews-create-mapping) |
| 74 | (search . nnweb-dejanewsold-search) | |
| 16409b0b | 75 | (address . "http://www.deja.com/dnquery.xp") |
| eec82323 LMI |
76 | (identifier . nnweb-dejanews-identity)) |
| 77 | (reference | |
| 78 | (article . nnweb-reference-wash-article) | |
| 79 | (map . nnweb-reference-create-mapping) | |
| 80 | (search . nnweb-reference-search) | |
| 81 | (address . "http://www.reference.com/cgi-bin/pn/go") | |
| 82 | (identifier . identity)) | |
| 83 | (altavista | |
| 84 | (article . nnweb-altavista-wash-article) | |
| 85 | (map . nnweb-altavista-create-mapping) | |
| 86 | (search . nnweb-altavista-search) | |
| 87 | (address . "http://www.altavista.digital.com/cgi-bin/query") | |
| 88 | (id . "/cgi-bin/news?id@%s") | |
| 89 | (identifier . identity))) | |
| 90 | "Type-definition alist.") | |
| 91 | ||
| 92 | (defvoo nnweb-search nil | |
| 93 | "Search string to feed to DejaNews.") | |
| 94 | ||
| 6748645f | 95 | (defvoo nnweb-max-hits 999 |
| eec82323 LMI |
96 | "Maximum number of hits to display.") |
| 97 | ||
| 98 | (defvoo nnweb-ephemeral-p nil | |
| 99 | "Whether this nnweb server is ephemeral.") | |
| 100 | ||
| 101 | ;;; Internal variables | |
| 102 | ||
| 103 | (defvoo nnweb-articles nil) | |
| 104 | (defvoo nnweb-buffer nil) | |
| 105 | (defvoo nnweb-group-alist nil) | |
| 106 | (defvoo nnweb-group nil) | |
| 107 | (defvoo nnweb-hashtb nil) | |
| 108 | ||
| 109 | ;;; Interface functions | |
| 110 | ||
| 111 | (nnoo-define-basics nnweb) | |
| 112 | ||
| 113 | (deffoo nnweb-retrieve-headers (articles &optional group server fetch-old) | |
| 114 | (nnweb-possibly-change-server group server) | |
| 115 | (save-excursion | |
| 116 | (set-buffer nntp-server-buffer) | |
| 117 | (erase-buffer) | |
| 118 | (let (article header) | |
| 16409b0b GM |
119 | (mm-with-unibyte-current-buffer |
| 120 | (while (setq article (pop articles)) | |
| 121 | (when (setq header (cadr (assq article nnweb-articles))) | |
| 122 | (nnheader-insert-nov header)))) | |
| eec82323 LMI |
123 | 'nov))) |
| 124 | ||
| 125 | (deffoo nnweb-request-scan (&optional group server) | |
| 126 | (nnweb-possibly-change-server group server) | |
| eec82323 LMI |
127 | (funcall (nnweb-definition 'map)) |
| 128 | (unless nnweb-ephemeral-p | |
| 129 | (nnweb-write-active) | |
| 130 | (nnweb-write-overview group))) | |
| 131 | ||
| 132 | (deffoo nnweb-request-group (group &optional server dont-check) | |
| 133 | (nnweb-possibly-change-server nil server) | |
| 134 | (when (and group | |
| 135 | (not (equal group nnweb-group)) | |
| 136 | (not nnweb-ephemeral-p)) | |
| 137 | (let ((info (assoc group nnweb-group-alist))) | |
| 16409b0b GM |
138 | (when info |
| 139 | (setq nnweb-group group) | |
| 140 | (setq nnweb-type (nth 2 info)) | |
| 141 | (setq nnweb-search (nth 3 info)) | |
| 142 | (unless dont-check | |
| 143 | (nnweb-read-overview group))))) | |
| eec82323 LMI |
144 | (cond |
| 145 | ((not nnweb-articles) | |
| 146 | (nnheader-report 'nnweb "No matching articles")) | |
| 147 | (t | |
| 148 | (let ((active (if nnweb-ephemeral-p | |
| 149 | (cons (caar nnweb-articles) | |
| 150 | (caar (last nnweb-articles))) | |
| 151 | (cadr (assoc group nnweb-group-alist))))) | |
| 152 | (nnheader-report 'nnweb "Opened group %s" group) | |
| 153 | (nnheader-insert | |
| 154 | "211 %d %d %d %s\n" (length nnweb-articles) | |
| 155 | (car active) (cdr active) group))))) | |
| 156 | ||
| 157 | (deffoo nnweb-close-group (group &optional server) | |
| 158 | (nnweb-possibly-change-server group server) | |
| 159 | (when (gnus-buffer-live-p nnweb-buffer) | |
| 160 | (save-excursion | |
| 161 | (set-buffer nnweb-buffer) | |
| 162 | (set-buffer-modified-p nil) | |
| 163 | (kill-buffer nnweb-buffer))) | |
| 164 | t) | |
| 165 | ||
| 166 | (deffoo nnweb-request-article (article &optional group server buffer) | |
| 167 | (nnweb-possibly-change-server group server) | |
| 168 | (save-excursion | |
| 169 | (set-buffer (or buffer nntp-server-buffer)) | |
| 170 | (let* ((header (cadr (assq article nnweb-articles))) | |
| 171 | (url (and header (mail-header-xref header)))) | |
| 172 | (when (or (and url | |
| 16409b0b GM |
173 | (mm-with-unibyte-current-buffer |
| 174 | (nnweb-fetch-url url))) | |
| eec82323 LMI |
175 | (and (stringp article) |
| 176 | (nnweb-definition 'id t) | |
| 177 | (let ((fetch (nnweb-definition 'id)) | |
| 178 | art) | |
| 179 | (when (string-match "^<\\(.*\\)>$" article) | |
| 180 | (setq art (match-string 1 article))) | |
| 181 | (and fetch | |
| 182 | art | |
| 16409b0b GM |
183 | (mm-with-unibyte-current-buffer |
| 184 | (nnweb-fetch-url | |
| 185 | (format fetch article))))))) | |
| eec82323 LMI |
186 | (unless nnheader-callback-function |
| 187 | (funcall (nnweb-definition 'article)) | |
| 188 | (nnweb-decode-entities)) | |
| 189 | (nnheader-report 'nnweb "Fetched article %s" article) | |
| 16409b0b | 190 | (cons group (and (numberp article) article)))))) |
| eec82323 LMI |
191 | |
| 192 | (deffoo nnweb-close-server (&optional server) | |
| 193 | (when (and (nnweb-server-opened server) | |
| 194 | (gnus-buffer-live-p nnweb-buffer)) | |
| 195 | (save-excursion | |
| 196 | (set-buffer nnweb-buffer) | |
| 197 | (set-buffer-modified-p nil) | |
| 198 | (kill-buffer nnweb-buffer))) | |
| 199 | (nnoo-close-server 'nnweb server)) | |
| 200 | ||
| 201 | (deffoo nnweb-request-list (&optional server) | |
| 202 | (nnweb-possibly-change-server nil server) | |
| 203 | (save-excursion | |
| 204 | (set-buffer nntp-server-buffer) | |
| 205 | (nnmail-generate-active nnweb-group-alist) | |
| 206 | t)) | |
| 207 | ||
| 208 | (deffoo nnweb-request-update-info (group info &optional server) | |
| 16409b0b | 209 | (nnweb-possibly-change-server group server)) |
| eec82323 LMI |
210 | |
| 211 | (deffoo nnweb-asynchronous-p () | |
| 212 | t) | |
| 213 | ||
| 214 | (deffoo nnweb-request-create-group (group &optional server args) | |
| 215 | (nnweb-possibly-change-server nil server) | |
| 216 | (nnweb-request-delete-group group) | |
| 217 | (push `(,group ,(cons 1 0) ,@args) nnweb-group-alist) | |
| 218 | (nnweb-write-active) | |
| 219 | t) | |
| 220 | ||
| 221 | (deffoo nnweb-request-delete-group (group &optional force server) | |
| 222 | (nnweb-possibly-change-server group server) | |
| 16409b0b GM |
223 | (gnus-pull group nnweb-group-alist t) |
| 224 | (nnweb-write-active) | |
| eec82323 LMI |
225 | (gnus-delete-file (nnweb-overview-file group)) |
| 226 | t) | |
| 227 | ||
| 228 | (nnoo-define-skeleton nnweb) | |
| 229 | ||
| 230 | ;;; Internal functions | |
| 231 | ||
| 232 | (defun nnweb-read-overview (group) | |
| 233 | "Read the overview of GROUP and build the map." | |
| 234 | (when (file-exists-p (nnweb-overview-file group)) | |
| 16409b0b | 235 | (mm-with-unibyte-buffer |
| eec82323 LMI |
236 | (nnheader-insert-file-contents (nnweb-overview-file group)) |
| 237 | (goto-char (point-min)) | |
| 238 | (let (header) | |
| 239 | (while (not (eobp)) | |
| 240 | (setq header (nnheader-parse-nov)) | |
| 241 | (forward-line 1) | |
| 242 | (push (list (mail-header-number header) | |
| 243 | header (mail-header-xref header)) | |
| 244 | nnweb-articles) | |
| 245 | (nnweb-set-hashtb header (car nnweb-articles))))))) | |
| 246 | ||
| 247 | (defun nnweb-write-overview (group) | |
| 248 | "Write the overview file for GROUP." | |
| 16409b0b | 249 | (with-temp-file (nnweb-overview-file group) |
| eec82323 LMI |
250 | (let ((articles nnweb-articles)) |
| 251 | (while articles | |
| 252 | (nnheader-insert-nov (cadr (pop articles))))))) | |
| 253 | ||
| 254 | (defun nnweb-set-hashtb (header data) | |
| 255 | (gnus-sethash (nnweb-identifier (mail-header-xref header)) | |
| 256 | data nnweb-hashtb)) | |
| 257 | ||
| 258 | (defun nnweb-get-hashtb (url) | |
| 259 | (gnus-gethash (nnweb-identifier url) nnweb-hashtb)) | |
| 260 | ||
| 261 | (defun nnweb-identifier (ident) | |
| 262 | (funcall (nnweb-definition 'identifier) ident)) | |
| 263 | ||
| 264 | (defun nnweb-overview-file (group) | |
| 265 | "Return the name of the overview file of GROUP." | |
| 266 | (nnheader-concat nnweb-directory group ".overview")) | |
| 267 | ||
| 268 | (defun nnweb-write-active () | |
| 269 | "Save the active file." | |
| 16409b0b GM |
270 | (gnus-make-directory nnweb-directory) |
| 271 | (with-temp-file (nnheader-concat nnweb-directory "active") | |
| eec82323 LMI |
272 | (prin1 `(setq nnweb-group-alist ',nnweb-group-alist) (current-buffer)))) |
| 273 | ||
| 274 | (defun nnweb-read-active () | |
| 275 | "Read the active file." | |
| 276 | (load (nnheader-concat nnweb-directory "active") t t t)) | |
| 277 | ||
| 278 | (defun nnweb-definition (type &optional noerror) | |
| 279 | "Return the definition of TYPE." | |
| 280 | (let ((def (cdr (assq type (assq nnweb-type nnweb-type-definition))))) | |
| 281 | (when (and (not def) | |
| 282 | (not noerror)) | |
| 283 | (error "Undefined definition %s" type)) | |
| 284 | def)) | |
| 285 | ||
| 286 | (defun nnweb-possibly-change-server (&optional group server) | |
| 287 | (nnweb-init server) | |
| 288 | (when server | |
| 289 | (unless (nnweb-server-opened server) | |
| 290 | (nnweb-open-server server))) | |
| 291 | (unless nnweb-group-alist | |
| 292 | (nnweb-read-active)) | |
| 293 | (when group | |
| 294 | (when (and (not nnweb-ephemeral-p) | |
| 295 | (not (equal group nnweb-group))) | |
| 16409b0b | 296 | (setq nnweb-hashtb (gnus-make-hashtable 4095)) |
| eec82323 LMI |
297 | (nnweb-request-group group nil t)))) |
| 298 | ||
| 299 | (defun nnweb-init (server) | |
| 300 | "Initialize buffers and such." | |
| 301 | (unless (gnus-buffer-live-p nnweb-buffer) | |
| 302 | (setq nnweb-buffer | |
| 303 | (save-excursion | |
| 16409b0b GM |
304 | (mm-with-unibyte |
| 305 | (nnheader-set-temp-buffer | |
| 306 | (format " *nnweb %s %s %s*" | |
| 307 | nnweb-type nnweb-search server)) | |
| 308 | (current-buffer)))))) | |
| eec82323 LMI |
309 | |
| 310 | (defun nnweb-fetch-url (url) | |
| 16409b0b GM |
311 | (let (buf) |
| 312 | (save-excursion | |
| 313 | (if (not nnheader-callback-function) | |
| 314 | (progn | |
| 315 | (with-temp-buffer | |
| 316 | (mm-enable-multibyte) | |
| 317 | (let ((coding-system-for-read 'binary) | |
| 318 | (coding-system-for-write 'binary) | |
| 319 | (default-process-coding-system 'binary)) | |
| 320 | (nnweb-insert url)) | |
| 321 | (setq buf (buffer-string))) | |
| eec82323 | 322 | (erase-buffer) |
| 16409b0b GM |
323 | (insert buf) |
| 324 | t) | |
| 325 | (nnweb-url-retrieve-asynch | |
| 326 | url 'nnweb-callback (current-buffer) nnheader-callback-function) | |
| 327 | t)))) | |
| eec82323 LMI |
328 | |
| 329 | (defun nnweb-callback (buffer callback) | |
| 330 | (when (gnus-buffer-live-p url-working-buffer) | |
| 331 | (save-excursion | |
| 332 | (set-buffer url-working-buffer) | |
| 333 | (funcall (nnweb-definition 'article)) | |
| 334 | (nnweb-decode-entities) | |
| 335 | (set-buffer buffer) | |
| 336 | (goto-char (point-max)) | |
| 337 | (insert-buffer-substring url-working-buffer)) | |
| 338 | (funcall callback t) | |
| 339 | (gnus-kill-buffer url-working-buffer))) | |
| 340 | ||
| 341 | (defun nnweb-url-retrieve-asynch (url callback &rest data) | |
| 342 | (let ((url-request-method "GET") | |
| 343 | (old-asynch url-be-asynchronous) | |
| 344 | (url-request-data nil) | |
| 345 | (url-request-extra-headers nil) | |
| 346 | (url-working-buffer (generate-new-buffer-name " *nnweb*"))) | |
| 347 | (setq-default url-be-asynchronous t) | |
| 348 | (save-excursion | |
| 349 | (set-buffer (get-buffer-create url-working-buffer)) | |
| 350 | (setq url-current-callback-data data | |
| 351 | url-be-asynchronous t | |
| 352 | url-current-callback-func callback) | |
| cd942edc | 353 | (url-retrieve url nil)) |
| eec82323 LMI |
354 | (setq-default url-be-asynchronous old-asynch))) |
| 355 | ||
| cd942edc DL |
356 | (if (fboundp 'url-retrieve-synchronously) |
| 357 | (defun nnweb-url-retrieve-asynch (url callback &rest data) | |
| 358 | (url-retrieve url callback data))) | |
| 359 | ||
| eec82323 LMI |
360 | ;;; |
| 361 | ;;; DejaNews functions. | |
| 362 | ;;; | |
| 363 | ||
| 364 | (defun nnweb-dejanews-create-mapping () | |
| 365 | "Perform the search and create an number-to-url alist." | |
| 366 | (save-excursion | |
| 367 | (set-buffer nnweb-buffer) | |
| 368 | (erase-buffer) | |
| 369 | (when (funcall (nnweb-definition 'search) nnweb-search) | |
| 370 | (let ((i 0) | |
| 371 | (more t) | |
| 372 | (case-fold-search t) | |
| 373 | (active (or (cadr (assoc nnweb-group nnweb-group-alist)) | |
| 374 | (cons 1 0))) | |
| 16409b0b GM |
375 | subject date from |
| 376 | map url parse a table group text) | |
| eec82323 LMI |
377 | (while more |
| 378 | ;; Go through all the article hits on this page. | |
| 379 | (goto-char (point-min)) | |
| 16409b0b GM |
380 | (setq parse (w3-parse-buffer (current-buffer)) |
| 381 | table (nth 1 (nnweb-parse-find-all 'table parse))) | |
| 382 | (dolist (row (nth 2 (car (nth 2 table)))) | |
| 383 | (setq a (nnweb-parse-find 'a row) | |
| 384 | url (cdr (assq 'href (nth 1 a))) | |
| 385 | text (nreverse (nnweb-text row))) | |
| 386 | (when a | |
| 387 | (setq subject (nth 4 text) | |
| 388 | group (nth 2 text) | |
| 389 | date (nth 1 text) | |
| 390 | from (nth 0 text)) | |
| 391 | (if (string-match "\\([0-9]+\\)/\\([0-9]+\\)/\\([0-9]+\\)" date) | |
| 392 | (setq date (format "%s %s 00:00:00 %s" | |
| 393 | (car (rassq (string-to-number | |
| 394 | (match-string 2 date)) | |
| 395 | parse-time-months)) | |
| 396 | (match-string 3 date) | |
| 397 | (match-string 1 date))) | |
| 398 | (setq date "Jan 1 00:00:00 0000")) | |
| 399 | (incf i) | |
| 400 | (setq url (concat url "&fmt=text")) | |
| 401 | (when (string-match "&context=[^&]+" url) | |
| 402 | (setq url (replace-match "" t t url))) | |
| 403 | (unless (nnweb-get-hashtb url) | |
| 404 | (push | |
| 405 | (list | |
| 406 | (incf (cdr active)) | |
| 407 | (make-full-mail-header | |
| 408 | (cdr active) (concat subject " (" group ")") from date | |
| 409 | (concat "<" (nnweb-identifier url) "@dejanews>") | |
| 410 | nil 0 0 url)) | |
| 411 | map) | |
| 412 | (nnweb-set-hashtb (cadar map) (car map))))) | |
| eec82323 | 413 | ;; See whether there is a "Get next 20 hits" button here. |
| 16409b0b | 414 | (goto-char (point-min)) |
| eec82323 | 415 | (if (or (not (re-search-forward |
| 6748645f | 416 | "HREF=\"\\([^\"]+\\)\"[<>b]+Next result" nil t)) |
| eec82323 LMI |
417 | (>= i nnweb-max-hits)) |
| 418 | (setq more nil) | |
| 419 | ;; Yup -- fetch it. | |
| 420 | (setq more (match-string 1)) | |
| 421 | (erase-buffer) | |
| 422 | (url-insert-file-contents more))) | |
| 423 | ;; Return the articles in the right order. | |
| 424 | (setq nnweb-articles | |
| 6748645f | 425 | (sort (nconc nnweb-articles map) 'car-less-than-car)))))) |
| eec82323 | 426 | |
| eec82323 | 427 | (defun nnweb-dejanews-search (search) |
| 16409b0b GM |
428 | (nnweb-insert |
| 429 | (concat | |
| 430 | (nnweb-definition 'address) | |
| 431 | "?" | |
| 432 | (nnweb-encode-www-form-urlencoded | |
| 433 | `(("ST" . "PS") | |
| 434 | ("svcclass" . "dnyr") | |
| 435 | ("QRY" . ,search) | |
| 436 | ("defaultOp" . "AND") | |
| 437 | ("DBS" . "1") | |
| 438 | ("OP" . "dnquery.xp") | |
| 439 | ("LNG" . "ALL") | |
| 440 | ("maxhits" . "100") | |
| 441 | ("threaded" . "0") | |
| 442 | ("format" . "verbose2") | |
| 443 | ("showsort" . "date") | |
| 444 | ("agesign" . "1") | |
| 445 | ("ageweight" . "1"))))) | |
| 6748645f LMI |
446 | t) |
| 447 | ||
| 448 | (defun nnweb-dejanewsold-search (search) | |
| 449 | (nnweb-fetch-form | |
| 450 | (nnweb-definition 'address) | |
| 451 | `(("query" . ,search) | |
| 452 | ("defaultOp" . "AND") | |
| 453 | ("svcclass" . "dnold") | |
| 454 | ("maxhits" . "100") | |
| 455 | ("format" . "verbose2") | |
| eec82323 | 456 | ("threaded" . "0") |
| 6748645f | 457 | ("showsort" . "date") |
| eec82323 LMI |
458 | ("agesign" . "1") |
| 459 | ("ageweight" . "1"))) | |
| 460 | t) | |
| 461 | ||
| 462 | (defun nnweb-dejanews-identity (url) | |
| 463 | "Return an unique identifier based on URL." | |
| 16409b0b | 464 | (if (string-match "AN=\\([0-9]+\\)" url) |
| eec82323 LMI |
465 | (match-string 1 url) |
| 466 | url)) | |
| 467 | ||
| 468 | ;;; | |
| 469 | ;;; InReference | |
| 470 | ;;; | |
| 471 | ||
| 472 | (defun nnweb-reference-create-mapping () | |
| 473 | "Perform the search and create an number-to-url alist." | |
| 474 | (save-excursion | |
| 475 | (set-buffer nnweb-buffer) | |
| 476 | (erase-buffer) | |
| 477 | (when (funcall (nnweb-definition 'search) nnweb-search) | |
| 478 | (let ((i 0) | |
| 479 | (more t) | |
| 480 | (case-fold-search t) | |
| 481 | (active (or (cadr (assoc nnweb-group nnweb-group-alist)) | |
| 482 | (cons 1 0))) | |
| 483 | Subject Score Date Newsgroups From Message-ID | |
| 484 | map url) | |
| 485 | (while more | |
| 486 | ;; Go through all the article hits on this page. | |
| 487 | (goto-char (point-min)) | |
| 488 | (search-forward "</pre><hr>" nil t) | |
| 489 | (delete-region (point-min) (point)) | |
| eec82323 LMI |
490 | (goto-char (point-min)) |
| 491 | (while (re-search-forward "^ +[0-9]+\\." nil t) | |
| 492 | (narrow-to-region | |
| 493 | (point) | |
| 494 | (if (re-search-forward "^$" nil t) | |
| 495 | (match-beginning 0) | |
| 496 | (point-max))) | |
| 497 | (goto-char (point-min)) | |
| 498 | (when (looking-at ".*href=\"\\([^\"]+\\)\"") | |
| 499 | (setq url (match-string 1))) | |
| 500 | (nnweb-remove-markup) | |
| 501 | (goto-char (point-min)) | |
| 502 | (while (search-forward "\t" nil t) | |
| 503 | (replace-match " ")) | |
| 504 | (goto-char (point-min)) | |
| 505 | (while (re-search-forward "^\\([^:]+\\): \\(.*\\)$" nil t) | |
| 506 | (set (intern (match-string 1)) (match-string 2))) | |
| 507 | (widen) | |
| 508 | (search-forward "</pre>" nil t) | |
| 509 | (incf i) | |
| 510 | (unless (nnweb-get-hashtb url) | |
| 511 | (push | |
| 512 | (list | |
| 513 | (incf (cdr active)) | |
| 514 | (make-full-mail-header | |
| 515 | (cdr active) (concat "(" Newsgroups ") " Subject) From Date | |
| 516 | Message-ID | |
| 517 | nil 0 (string-to-int Score) url)) | |
| 518 | map) | |
| 519 | (nnweb-set-hashtb (cadar map) (car map)))) | |
| 520 | (setq more nil)) | |
| 521 | ;; Return the articles in the right order. | |
| 522 | (setq nnweb-articles | |
| 6748645f | 523 | (sort (nconc nnweb-articles map) 'car-less-than-car)))))) |
| eec82323 LMI |
524 | |
| 525 | (defun nnweb-reference-wash-article () | |
| 526 | (let ((case-fold-search t)) | |
| 527 | (goto-char (point-min)) | |
| 528 | (re-search-forward "^</center><hr>" nil t) | |
| 529 | (delete-region (point-min) (point)) | |
| 530 | (search-forward "<pre>" nil t) | |
| 531 | (forward-line -1) | |
| 532 | (let ((body (point-marker))) | |
| 533 | (search-forward "</pre>" nil t) | |
| 534 | (delete-region (point) (point-max)) | |
| 535 | (nnweb-remove-markup) | |
| 536 | (goto-char (point-min)) | |
| 537 | (while (looking-at " *$") | |
| 538 | (gnus-delete-line)) | |
| 539 | (narrow-to-region (point-min) body) | |
| 540 | (while (and (re-search-forward "^$" nil t) | |
| 541 | (not (eobp))) | |
| 542 | (gnus-delete-line)) | |
| 543 | (goto-char (point-min)) | |
| 544 | (while (looking-at "\\(^[^ ]+:\\) *") | |
| 545 | (replace-match "\\1 " t) | |
| 546 | (forward-line 1)) | |
| 547 | (goto-char (point-min)) | |
| 548 | (when (re-search-forward "^References:" nil t) | |
| 549 | (narrow-to-region | |
| 550 | (point) (if (re-search-forward "^$\\|^[^:]+:" nil t) | |
| 551 | (match-beginning 0) | |
| 552 | (point-max))) | |
| 553 | (goto-char (point-min)) | |
| 554 | (while (not (eobp)) | |
| 555 | (unless (looking-at "References") | |
| 556 | (insert "\t") | |
| 557 | (forward-line 1))) | |
| 558 | (goto-char (point-min)) | |
| 559 | (while (search-forward "," nil t) | |
| 560 | (replace-match " " t t))) | |
| 561 | (widen) | |
| 562 | (set-marker body nil)))) | |
| 563 | ||
| 564 | (defun nnweb-reference-search (search) | |
| a8151ef7 LMI |
565 | (url-insert-file-contents |
| 566 | (concat | |
| 567 | (nnweb-definition 'address) | |
| 568 | "?" | |
| 569 | (nnweb-encode-www-form-urlencoded | |
| 570 | `(("search" . "advanced") | |
| 571 | ("querytext" . ,search) | |
| 572 | ("subj" . "") | |
| 573 | ("name" . "") | |
| 574 | ("login" . "") | |
| 575 | ("host" . "") | |
| 576 | ("organization" . "") | |
| 577 | ("groups" . "") | |
| 578 | ("keywords" . "") | |
| 579 | ("choice" . "Search") | |
| 580 | ("startmonth" . "Jul") | |
| 581 | ("startday" . "25") | |
| 582 | ("startyear" . "1996") | |
| 583 | ("endmonth" . "Aug") | |
| 584 | ("endday" . "24") | |
| 585 | ("endyear" . "1996") | |
| 586 | ("mode" . "Quick") | |
| 587 | ("verbosity" . "Verbose") | |
| 588 | ("ranking" . "Relevance") | |
| 589 | ("first" . "1") | |
| 590 | ("last" . "25") | |
| 591 | ("score" . "50"))))) | |
| 592 | (setq buffer-file-name nil) | |
| eec82323 LMI |
593 | t) |
| 594 | ||
| 595 | ;;; | |
| 596 | ;;; Alta Vista | |
| 597 | ;;; | |
| 598 | ||
| 599 | (defun nnweb-altavista-create-mapping () | |
| 600 | "Perform the search and create an number-to-url alist." | |
| 601 | (save-excursion | |
| 602 | (set-buffer nnweb-buffer) | |
| 603 | (erase-buffer) | |
| 604 | (let ((part 0)) | |
| 605 | (when (funcall (nnweb-definition 'search) nnweb-search part) | |
| 606 | (let ((i 0) | |
| 607 | (more t) | |
| 608 | (case-fold-search t) | |
| 609 | (active (or (cadr (assoc nnweb-group nnweb-group-alist)) | |
| 610 | (cons 1 0))) | |
| 611 | subject date from id group | |
| 612 | map url) | |
| 613 | (while more | |
| 614 | ;; Go through all the article hits on this page. | |
| 615 | (goto-char (point-min)) | |
| 616 | (search-forward "<dt>" nil t) | |
| 617 | (delete-region (point-min) (match-beginning 0)) | |
| 618 | (goto-char (point-min)) | |
| 619 | (while (search-forward "<dt>" nil t) | |
| 620 | (replace-match "\n<blubb>")) | |
| 621 | (nnweb-decode-entities) | |
| 622 | (goto-char (point-min)) | |
| 623 | (while (re-search-forward "<blubb>.*href=\"\\([^\"]+\\)\"><strong>\\([^>]*\\)</strong></a><dd>\\([^-]+\\)- <b>\\([^<]+\\)<.*href=\"news:\\([^\"]+\\)\">.*\">\\(.+\\)</a><P>" | |
| 624 | nil t) | |
| 625 | (setq url (match-string 1) | |
| 626 | subject (match-string 2) | |
| 627 | date (match-string 3) | |
| 628 | group (match-string 4) | |
| 629 | id (concat "<" (match-string 5) ">") | |
| 630 | from (match-string 6)) | |
| 631 | (incf i) | |
| 632 | (unless (nnweb-get-hashtb url) | |
| 633 | (push | |
| 634 | (list | |
| 635 | (incf (cdr active)) | |
| 636 | (make-full-mail-header | |
| 637 | (cdr active) (concat "(" group ") " subject) from date | |
| 638 | id nil 0 0 url)) | |
| 639 | map) | |
| 640 | (nnweb-set-hashtb (cadar map) (car map)))) | |
| 641 | ;; See if we want more. | |
| 642 | (when (or (not nnweb-articles) | |
| 643 | (>= i nnweb-max-hits) | |
| 644 | (not (funcall (nnweb-definition 'search) | |
| 645 | nnweb-search (incf part)))) | |
| 646 | (setq more nil))) | |
| 647 | ;; Return the articles in the right order. | |
| 648 | (setq nnweb-articles | |
| 6748645f | 649 | (sort (nconc nnweb-articles map) 'car-less-than-car))))))) |
| eec82323 LMI |
650 | |
| 651 | (defun nnweb-altavista-wash-article () | |
| 652 | (goto-char (point-min)) | |
| 653 | (let ((case-fold-search t)) | |
| 654 | (when (re-search-forward "^<strong>" nil t) | |
| 655 | (delete-region (point-min) (match-beginning 0))) | |
| 656 | (goto-char (point-min)) | |
| 657 | (while (looking-at "<strong>\\([^ ]+\\) +</strong> +\\(.*\\)$") | |
| 658 | (replace-match "\\1: \\2" t) | |
| 659 | (forward-line 1)) | |
| 660 | (when (re-search-backward "^References:" nil t) | |
| 661 | (narrow-to-region (point) (progn (forward-line 1) (point))) | |
| 662 | (goto-char (point-min)) | |
| 663 | (while (re-search-forward "<A.*\\?id@\\([^\"]+\\)\">[0-9]+</A>" nil t) | |
| 664 | (replace-match "<\\1> " t))) | |
| 665 | (widen) | |
| 666 | (nnweb-remove-markup))) | |
| 667 | ||
| 668 | (defun nnweb-altavista-search (search &optional part) | |
| a8151ef7 LMI |
669 | (url-insert-file-contents |
| 670 | (concat | |
| 671 | (nnweb-definition 'address) | |
| 672 | "?" | |
| 673 | (nnweb-encode-www-form-urlencoded | |
| 674 | `(("pg" . "aq") | |
| 675 | ("what" . "news") | |
| 676 | ,@(when part `(("stq" . ,(int-to-string (* part 30))))) | |
| 677 | ("fmt" . "d") | |
| 678 | ("q" . ,search) | |
| 679 | ("r" . "") | |
| 680 | ("d0" . "") | |
| 681 | ("d1" . ""))))) | |
| 682 | (setq buffer-file-name nil) | |
| 683 | t) | |
| eec82323 | 684 | |
| 16409b0b GM |
685 | ;;; |
| 686 | ;;; General web/w3 interface utility functions | |
| 687 | ;;; | |
| 688 | ||
| 689 | (defun nnweb-insert-html (parse) | |
| 690 | "Insert HTML based on a w3 parse tree." | |
| 691 | (if (stringp parse) | |
| 692 | (insert parse) | |
| 693 | (insert "<" (symbol-name (car parse)) " ") | |
| 694 | (insert (mapconcat | |
| 695 | (lambda (param) | |
| 696 | (concat (symbol-name (car param)) "=" | |
| 697 | (prin1-to-string | |
| 698 | (if (consp (cdr param)) | |
| 699 | (cadr param) | |
| 700 | (cdr param))))) | |
| 701 | (nth 1 parse) | |
| 702 | " ")) | |
| 703 | (insert ">\n") | |
| 704 | (mapcar 'nnweb-insert-html (nth 2 parse)) | |
| 705 | (insert "</" (symbol-name (car parse)) ">\n"))) | |
| 706 | ||
| 707 | (defun nnweb-encode-www-form-urlencoded (pairs) | |
| 708 | "Return PAIRS encoded for forms." | |
| 709 | (mapconcat | |
| 710 | (function | |
| 711 | (lambda (data) | |
| 712 | (concat (w3-form-encode-xwfu (car data)) "=" | |
| 713 | (w3-form-encode-xwfu (cdr data))))) | |
| 714 | pairs "&")) | |
| 715 | ||
| 716 | (defun nnweb-fetch-form (url pairs) | |
| 717 | "Fetch a form from URL with PAIRS as the data using the POST method." | |
| 718 | (let ((url-request-data (nnweb-encode-www-form-urlencoded pairs)) | |
| 719 | (url-request-method "POST") | |
| 720 | (url-request-extra-headers | |
| 721 | '(("Content-type" . "application/x-www-form-urlencoded")))) | |
| 722 | (url-insert-file-contents url) | |
| 723 | (setq buffer-file-name nil)) | |
| 724 | t) | |
| 725 | ||
| 726 | (defun nnweb-decode-entities () | |
| 727 | "Decode all HTML entities." | |
| 728 | (goto-char (point-min)) | |
| 729 | (while (re-search-forward "&\\(#[0-9]+\\|[a-z]+\\);" nil t) | |
| cd942edc | 730 | (let ((elem (if (eq (aref (match-string 1) 0) ?\#) |
| 16409b0b GM |
731 | (let ((c |
| 732 | (string-to-number (substring | |
| 733 | (match-string 1) 1)))) | |
| 734 | (if (mm-char-or-char-int-p c) c 32)) | |
| 735 | (or (cdr (assq (intern (match-string 1)) | |
| 736 | w3-html-entities)) | |
| cd942edc DL |
737 | ?#)))) |
| 738 | (unless (stringp elem) | |
| 739 | (setq elem (char-to-string elem))) | |
| 740 | (replace-match elem t t)))) | |
| 16409b0b GM |
741 | |
| 742 | (defun nnweb-decode-entities-string (str) | |
| 743 | (with-temp-buffer | |
| 744 | (insert str) | |
| 745 | (nnweb-decode-entities) | |
| 746 | (buffer-substring (point-min) (point-max)))) | |
| 747 | ||
| 748 | (defun nnweb-remove-markup () | |
| 749 | "Remove all HTML markup, leaving just plain text." | |
| 750 | (goto-char (point-min)) | |
| 751 | (while (search-forward "<!--" nil t) | |
| 752 | (delete-region (match-beginning 0) | |
| 753 | (or (search-forward "-->" nil t) | |
| 754 | (point-max)))) | |
| 755 | (goto-char (point-min)) | |
| 756 | (while (re-search-forward "<[^>]+>" nil t) | |
| 757 | (replace-match "" t t))) | |
| 758 | ||
| 759 | (defun nnweb-insert (url &optional follow-refresh) | |
| 760 | "Insert the contents from an URL in the current buffer. | |
| 761 | If FOLLOW-REFRESH is non-nil, redirect refresh url in META." | |
| 762 | (let ((name buffer-file-name)) | |
| 763 | (if follow-refresh | |
| 764 | (save-restriction | |
| 765 | (narrow-to-region (point) (point)) | |
| 766 | (url-insert-file-contents url) | |
| 767 | (goto-char (point-min)) | |
| 768 | (when (re-search-forward | |
| 769 | "<meta[ \t\r\n]*http-equiv=\"Refresh\"[^>]*URL=\\([^\"]+\\)\"" nil t) | |
| 770 | (let ((url (match-string 1))) | |
| 771 | (delete-region (point-min) (point-max)) | |
| 772 | (nnweb-insert url t)))) | |
| 773 | (url-insert-file-contents url)) | |
| 774 | (setq buffer-file-name name))) | |
| 775 | ||
| 776 | (defun nnweb-parse-find (type parse &optional maxdepth) | |
| 777 | "Find the element of TYPE in PARSE." | |
| 778 | (catch 'found | |
| 779 | (nnweb-parse-find-1 type parse maxdepth))) | |
| 780 | ||
| 781 | (defun nnweb-parse-find-1 (type contents maxdepth) | |
| 782 | (when (or (null maxdepth) | |
| 783 | (not (zerop maxdepth))) | |
| 784 | (when (consp contents) | |
| 785 | (when (eq (car contents) type) | |
| 786 | (throw 'found contents)) | |
| 787 | (when (listp (cdr contents)) | |
| 788 | (dolist (element contents) | |
| 789 | (when (consp element) | |
| 790 | (nnweb-parse-find-1 type element | |
| 791 | (and maxdepth (1- maxdepth))))))))) | |
| 792 | ||
| 793 | (defun nnweb-parse-find-all (type parse) | |
| 794 | "Find all elements of TYPE in PARSE." | |
| 795 | (catch 'found | |
| 796 | (nnweb-parse-find-all-1 type parse))) | |
| 797 | ||
| 798 | (defun nnweb-parse-find-all-1 (type contents) | |
| 799 | (let (result) | |
| 800 | (when (consp contents) | |
| 801 | (if (eq (car contents) type) | |
| 802 | (push contents result) | |
| 803 | (when (listp (cdr contents)) | |
| 804 | (dolist (element contents) | |
| 805 | (when (consp element) | |
| 806 | (setq result | |
| 807 | (nconc result (nnweb-parse-find-all-1 type element)))))))) | |
| 808 | result)) | |
| 809 | ||
| 810 | (defvar nnweb-text) | |
| 811 | (defun nnweb-text (parse) | |
| 812 | "Return a list of text contents in PARSE." | |
| 813 | (let ((nnweb-text nil)) | |
| 814 | (nnweb-text-1 parse) | |
| 815 | (nreverse nnweb-text))) | |
| 816 | ||
| 817 | (defun nnweb-text-1 (contents) | |
| 818 | (dolist (element contents) | |
| 819 | (if (stringp element) | |
| 820 | (push element nnweb-text) | |
| 821 | (when (and (consp element) | |
| 822 | (listp (cdr element))) | |
| 823 | (nnweb-text-1 element))))) | |
| 824 | ||
| eec82323 LMI |
825 | (provide 'nnweb) |
| 826 | ||
| 827 | ;;; nnweb.el ends here |