HCoop / bpt / emacs.git / blame
| Commit | Line | Data |
|---|---|---|
| eec82323 LMI |
1 | ;;; nnweb.el --- retrieving articles via web search engines |
| 2 | ;; Copyright (C) 1996,97 Free Software Foundation, Inc. | |
| 3 | ||
| 4 | ;; Author: Lars Magne Ingebrigtsen <larsi@ifi.uio.no> | |
| 5 | ;; Keywords: news | |
| 6 | ||
| 7 | ;; This file is part of GNU Emacs. | |
| 8 | ||
| 9 | ;; GNU Emacs is free software; you can redistribute it and/or modify | |
| 10 | ;; it under the terms of the GNU General Public License as published by | |
| 11 | ;; the Free Software Foundation; either version 2, or (at your option) | |
| 12 | ;; any later version. | |
| 13 | ||
| 14 | ;; GNU Emacs is distributed in the hope that it will be useful, | |
| 15 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 16 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 17 | ;; GNU General Public License for more details. | |
| 18 | ||
| 19 | ;; You should have received a copy of the GNU General Public License | |
| 20 | ;; along with GNU Emacs; see the file COPYING. If not, write to the | |
| 21 | ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
| 22 | ;; Boston, MA 02111-1307, USA. | |
| 23 | ||
| 24 | ;;; Commentary: | |
| 25 | ||
| 26 | ;; Note: You need to have `url' and `w3' installed for this | |
| 27 | ;; backend to work. | |
| 28 | ||
| 29 | ;;; Code: | |
| 30 | ||
| 5ab7173c RS |
31 | (eval-when-compile (require 'cl)) |
| 32 | ||
| eec82323 LMI |
33 | (require 'nnoo) |
| 34 | (require 'message) | |
| 35 | (require 'gnus-util) | |
| 36 | (require 'gnus) | |
| eec82323 | 37 | (require 'nnmail) |
| 0d5dc4a5 DL |
38 | (eval-when-compile |
| 39 | (ignore-errors | |
| 40 | (require 'w3) | |
| 41 | (require 'url) | |
| 42 | (require 'w3-forms))) | |
| 43 | ;; Report failure to find w3 at load time if appropriate. | |
| 44 | (eval '(progn | |
| 45 | (require 'w3) | |
| 46 | (require 'url) | |
| 47 | (require 'w3-forms))) | |
| eec82323 LMI |
48 | |
| 49 | (nnoo-declare nnweb) | |
| 50 | ||
| 51 | (defvoo nnweb-directory (nnheader-concat gnus-directory "nnweb/") | |
| 52 | "Where nnweb will save its files.") | |
| 53 | ||
| 54 | (defvoo nnweb-type 'dejanews | |
| 55 | "What search engine type is being used.") | |
| 56 | ||
| 57 | (defvar nnweb-type-definition | |
| 58 | '((dejanews | |
| 59 | (article . nnweb-dejanews-wash-article) | |
| 60 | (map . nnweb-dejanews-create-mapping) | |
| 61 | (search . nnweb-dejanews-search) | |
| 62 | (address . "http://xp9.dejanews.com/dnquery.xp") | |
| 63 | (identifier . nnweb-dejanews-identity)) | |
| 64 | (reference | |
| 65 | (article . nnweb-reference-wash-article) | |
| 66 | (map . nnweb-reference-create-mapping) | |
| 67 | (search . nnweb-reference-search) | |
| 68 | (address . "http://www.reference.com/cgi-bin/pn/go") | |
| 69 | (identifier . identity)) | |
| 70 | (altavista | |
| 71 | (article . nnweb-altavista-wash-article) | |
| 72 | (map . nnweb-altavista-create-mapping) | |
| 73 | (search . nnweb-altavista-search) | |
| 74 | (address . "http://www.altavista.digital.com/cgi-bin/query") | |
| 75 | (id . "/cgi-bin/news?id@%s") | |
| 76 | (identifier . identity))) | |
| 77 | "Type-definition alist.") | |
| 78 | ||
| 79 | (defvoo nnweb-search nil | |
| 80 | "Search string to feed to DejaNews.") | |
| 81 | ||
| 82 | (defvoo nnweb-max-hits 100 | |
| 83 | "Maximum number of hits to display.") | |
| 84 | ||
| 85 | (defvoo nnweb-ephemeral-p nil | |
| 86 | "Whether this nnweb server is ephemeral.") | |
| 87 | ||
| 88 | ;;; Internal variables | |
| 89 | ||
| 90 | (defvoo nnweb-articles nil) | |
| 91 | (defvoo nnweb-buffer nil) | |
| 92 | (defvoo nnweb-group-alist nil) | |
| 93 | (defvoo nnweb-group nil) | |
| 94 | (defvoo nnweb-hashtb nil) | |
| 95 | ||
| 96 | ;;; Interface functions | |
| 97 | ||
| 98 | (nnoo-define-basics nnweb) | |
| 99 | ||
| 100 | (deffoo nnweb-retrieve-headers (articles &optional group server fetch-old) | |
| 101 | (nnweb-possibly-change-server group server) | |
| 102 | (save-excursion | |
| 103 | (set-buffer nntp-server-buffer) | |
| 104 | (erase-buffer) | |
| 105 | (let (article header) | |
| 106 | (while (setq article (pop articles)) | |
| 107 | (when (setq header (cadr (assq article nnweb-articles))) | |
| 108 | (nnheader-insert-nov header))) | |
| 109 | 'nov))) | |
| 110 | ||
| 111 | (deffoo nnweb-request-scan (&optional group server) | |
| 112 | (nnweb-possibly-change-server group server) | |
| 113 | (setq nnweb-hashtb (gnus-make-hashtable 4095)) | |
| 114 | (funcall (nnweb-definition 'map)) | |
| 115 | (unless nnweb-ephemeral-p | |
| 116 | (nnweb-write-active) | |
| 117 | (nnweb-write-overview group))) | |
| 118 | ||
| 119 | (deffoo nnweb-request-group (group &optional server dont-check) | |
| 120 | (nnweb-possibly-change-server nil server) | |
| 121 | (when (and group | |
| 122 | (not (equal group nnweb-group)) | |
| 123 | (not nnweb-ephemeral-p)) | |
| 124 | (let ((info (assoc group nnweb-group-alist))) | |
| 125 | (setq nnweb-group group) | |
| 126 | (setq nnweb-type (nth 2 info)) | |
| 127 | (setq nnweb-search (nth 3 info)) | |
| 128 | (unless dont-check | |
| 129 | (nnweb-read-overview group)))) | |
| 130 | (cond | |
| 131 | ((not nnweb-articles) | |
| 132 | (nnheader-report 'nnweb "No matching articles")) | |
| 133 | (t | |
| 134 | (let ((active (if nnweb-ephemeral-p | |
| 135 | (cons (caar nnweb-articles) | |
| 136 | (caar (last nnweb-articles))) | |
| 137 | (cadr (assoc group nnweb-group-alist))))) | |
| 138 | (nnheader-report 'nnweb "Opened group %s" group) | |
| 139 | (nnheader-insert | |
| 140 | "211 %d %d %d %s\n" (length nnweb-articles) | |
| 141 | (car active) (cdr active) group))))) | |
| 142 | ||
| 143 | (deffoo nnweb-close-group (group &optional server) | |
| 144 | (nnweb-possibly-change-server group server) | |
| 145 | (when (gnus-buffer-live-p nnweb-buffer) | |
| 146 | (save-excursion | |
| 147 | (set-buffer nnweb-buffer) | |
| 148 | (set-buffer-modified-p nil) | |
| 149 | (kill-buffer nnweb-buffer))) | |
| 150 | t) | |
| 151 | ||
| 152 | (deffoo nnweb-request-article (article &optional group server buffer) | |
| 153 | (nnweb-possibly-change-server group server) | |
| 154 | (save-excursion | |
| 155 | (set-buffer (or buffer nntp-server-buffer)) | |
| 156 | (let* ((header (cadr (assq article nnweb-articles))) | |
| 157 | (url (and header (mail-header-xref header)))) | |
| 158 | (when (or (and url | |
| 159 | (nnweb-fetch-url url)) | |
| 160 | (and (stringp article) | |
| 161 | (nnweb-definition 'id t) | |
| 162 | (let ((fetch (nnweb-definition 'id)) | |
| 163 | art) | |
| 164 | (when (string-match "^<\\(.*\\)>$" article) | |
| 165 | (setq art (match-string 1 article))) | |
| 166 | (and fetch | |
| 167 | art | |
| 168 | (nnweb-fetch-url | |
| 169 | (format fetch article)))))) | |
| 170 | (unless nnheader-callback-function | |
| 171 | (funcall (nnweb-definition 'article)) | |
| 172 | (nnweb-decode-entities)) | |
| 173 | (nnheader-report 'nnweb "Fetched article %s" article) | |
| 174 | t)))) | |
| 175 | ||
| 176 | (deffoo nnweb-close-server (&optional server) | |
| 177 | (when (and (nnweb-server-opened server) | |
| 178 | (gnus-buffer-live-p nnweb-buffer)) | |
| 179 | (save-excursion | |
| 180 | (set-buffer nnweb-buffer) | |
| 181 | (set-buffer-modified-p nil) | |
| 182 | (kill-buffer nnweb-buffer))) | |
| 183 | (nnoo-close-server 'nnweb server)) | |
| 184 | ||
| 185 | (deffoo nnweb-request-list (&optional server) | |
| 186 | (nnweb-possibly-change-server nil server) | |
| 187 | (save-excursion | |
| 188 | (set-buffer nntp-server-buffer) | |
| 189 | (nnmail-generate-active nnweb-group-alist) | |
| 190 | t)) | |
| 191 | ||
| 192 | (deffoo nnweb-request-update-info (group info &optional server) | |
| 193 | (nnweb-possibly-change-server group server) | |
| 194 | ;;(setcar (cddr info) nil) | |
| 195 | ) | |
| 196 | ||
| 197 | (deffoo nnweb-asynchronous-p () | |
| 198 | t) | |
| 199 | ||
| 200 | (deffoo nnweb-request-create-group (group &optional server args) | |
| 201 | (nnweb-possibly-change-server nil server) | |
| 202 | (nnweb-request-delete-group group) | |
| 203 | (push `(,group ,(cons 1 0) ,@args) nnweb-group-alist) | |
| 204 | (nnweb-write-active) | |
| 205 | t) | |
| 206 | ||
| 207 | (deffoo nnweb-request-delete-group (group &optional force server) | |
| 208 | (nnweb-possibly-change-server group server) | |
| 209 | (gnus-delete-assoc group nnweb-group-alist) | |
| 210 | (gnus-delete-file (nnweb-overview-file group)) | |
| 211 | t) | |
| 212 | ||
| 213 | (nnoo-define-skeleton nnweb) | |
| 214 | ||
| 215 | ;;; Internal functions | |
| 216 | ||
| 217 | (defun nnweb-read-overview (group) | |
| 218 | "Read the overview of GROUP and build the map." | |
| 219 | (when (file-exists-p (nnweb-overview-file group)) | |
| 220 | (nnheader-temp-write nil | |
| 221 | (nnheader-insert-file-contents (nnweb-overview-file group)) | |
| 222 | (goto-char (point-min)) | |
| 223 | (let (header) | |
| 224 | (while (not (eobp)) | |
| 225 | (setq header (nnheader-parse-nov)) | |
| 226 | (forward-line 1) | |
| 227 | (push (list (mail-header-number header) | |
| 228 | header (mail-header-xref header)) | |
| 229 | nnweb-articles) | |
| 230 | (nnweb-set-hashtb header (car nnweb-articles))))))) | |
| 231 | ||
| 232 | (defun nnweb-write-overview (group) | |
| 233 | "Write the overview file for GROUP." | |
| 234 | (nnheader-temp-write (nnweb-overview-file group) | |
| 235 | (let ((articles nnweb-articles)) | |
| 236 | (while articles | |
| 237 | (nnheader-insert-nov (cadr (pop articles))))))) | |
| 238 | ||
| 239 | (defun nnweb-set-hashtb (header data) | |
| 240 | (gnus-sethash (nnweb-identifier (mail-header-xref header)) | |
| 241 | data nnweb-hashtb)) | |
| 242 | ||
| 243 | (defun nnweb-get-hashtb (url) | |
| 244 | (gnus-gethash (nnweb-identifier url) nnweb-hashtb)) | |
| 245 | ||
| 246 | (defun nnweb-identifier (ident) | |
| 247 | (funcall (nnweb-definition 'identifier) ident)) | |
| 248 | ||
| 249 | (defun nnweb-overview-file (group) | |
| 250 | "Return the name of the overview file of GROUP." | |
| 251 | (nnheader-concat nnweb-directory group ".overview")) | |
| 252 | ||
| 253 | (defun nnweb-write-active () | |
| 254 | "Save the active file." | |
| 255 | (nnheader-temp-write (nnheader-concat nnweb-directory "active") | |
| 256 | (prin1 `(setq nnweb-group-alist ',nnweb-group-alist) (current-buffer)))) | |
| 257 | ||
| 258 | (defun nnweb-read-active () | |
| 259 | "Read the active file." | |
| 260 | (load (nnheader-concat nnweb-directory "active") t t t)) | |
| 261 | ||
| 262 | (defun nnweb-definition (type &optional noerror) | |
| 263 | "Return the definition of TYPE." | |
| 264 | (let ((def (cdr (assq type (assq nnweb-type nnweb-type-definition))))) | |
| 265 | (when (and (not def) | |
| 266 | (not noerror)) | |
| 267 | (error "Undefined definition %s" type)) | |
| 268 | def)) | |
| 269 | ||
| 270 | (defun nnweb-possibly-change-server (&optional group server) | |
| 271 | (nnweb-init server) | |
| 272 | (when server | |
| 273 | (unless (nnweb-server-opened server) | |
| 274 | (nnweb-open-server server))) | |
| 275 | (unless nnweb-group-alist | |
| 276 | (nnweb-read-active)) | |
| 277 | (when group | |
| 278 | (when (and (not nnweb-ephemeral-p) | |
| 279 | (not (equal group nnweb-group))) | |
| 280 | (nnweb-request-group group nil t)))) | |
| 281 | ||
| 282 | (defun nnweb-init (server) | |
| 283 | "Initialize buffers and such." | |
| 284 | (unless (gnus-buffer-live-p nnweb-buffer) | |
| 285 | (setq nnweb-buffer | |
| 286 | (save-excursion | |
| 287 | (nnheader-set-temp-buffer | |
| 288 | (format " *nnweb %s %s %s*" nnweb-type nnweb-search server)))))) | |
| 289 | ||
| 290 | (defun nnweb-fetch-url (url) | |
| 291 | (save-excursion | |
| 292 | (if (not nnheader-callback-function) | |
| 293 | (let ((buf (current-buffer))) | |
| 294 | (save-excursion | |
| 295 | (set-buffer nnweb-buffer) | |
| 296 | (erase-buffer) | |
| a8151ef7 LMI |
297 | (url-insert-file-contents url) |
| 298 | (copy-to-buffer buf (point-min) (point-max)) | |
| 299 | t)) | |
| eec82323 LMI |
300 | (nnweb-url-retrieve-asynch |
| 301 | url 'nnweb-callback (current-buffer) nnheader-callback-function) | |
| 302 | t))) | |
| 303 | ||
| 304 | (defun nnweb-callback (buffer callback) | |
| 305 | (when (gnus-buffer-live-p url-working-buffer) | |
| 306 | (save-excursion | |
| 307 | (set-buffer url-working-buffer) | |
| 308 | (funcall (nnweb-definition 'article)) | |
| 309 | (nnweb-decode-entities) | |
| 310 | (set-buffer buffer) | |
| 311 | (goto-char (point-max)) | |
| 312 | (insert-buffer-substring url-working-buffer)) | |
| 313 | (funcall callback t) | |
| 314 | (gnus-kill-buffer url-working-buffer))) | |
| 315 | ||
| 316 | (defun nnweb-url-retrieve-asynch (url callback &rest data) | |
| 317 | (let ((url-request-method "GET") | |
| 318 | (old-asynch url-be-asynchronous) | |
| 319 | (url-request-data nil) | |
| 320 | (url-request-extra-headers nil) | |
| 321 | (url-working-buffer (generate-new-buffer-name " *nnweb*"))) | |
| 322 | (setq-default url-be-asynchronous t) | |
| 323 | (save-excursion | |
| 324 | (set-buffer (get-buffer-create url-working-buffer)) | |
| 325 | (setq url-current-callback-data data | |
| 326 | url-be-asynchronous t | |
| 327 | url-current-callback-func callback) | |
| 328 | (url-retrieve url)) | |
| 329 | (setq-default url-be-asynchronous old-asynch))) | |
| 330 | ||
| 331 | (defun nnweb-encode-www-form-urlencoded (pairs) | |
| 332 | "Return PAIRS encoded for forms." | |
| 333 | (mapconcat | |
| 334 | (function | |
| 335 | (lambda (data) | |
| 336 | (concat (w3-form-encode-xwfu (car data)) "=" | |
| 337 | (w3-form-encode-xwfu (cdr data))))) | |
| 338 | pairs "&")) | |
| 339 | ||
| 340 | (defun nnweb-fetch-form (url pairs) | |
| 341 | (let ((url-request-data (nnweb-encode-www-form-urlencoded pairs)) | |
| 342 | (url-request-method "POST") | |
| 343 | (url-request-extra-headers | |
| 344 | '(("Content-type" . "application/x-www-form-urlencoded")))) | |
| 345 | (url-insert-file-contents url) | |
| 346 | (setq buffer-file-name nil)) | |
| 347 | t) | |
| 348 | ||
| 349 | (defun nnweb-decode-entities () | |
| 350 | (goto-char (point-min)) | |
| 351 | (while (re-search-forward "&\\([a-z]+\\);" nil t) | |
| 352 | (replace-match (char-to-string (or (cdr (assq (intern (match-string 1)) | |
| a8151ef7 | 353 | w3-html-entities)) |
| eec82323 LMI |
354 | ?#)) |
| 355 | t t))) | |
| 356 | ||
| 357 | (defun nnweb-remove-markup () | |
| 358 | (goto-char (point-min)) | |
| 359 | (while (search-forward "<!--" nil t) | |
| 360 | (delete-region (match-beginning 0) | |
| 361 | (or (search-forward "-->" nil t) | |
| 362 | (point-max)))) | |
| 363 | (goto-char (point-min)) | |
| 364 | (while (re-search-forward "<[^>]+>" nil t) | |
| 365 | (replace-match "" t t))) | |
| 366 | ||
| 367 | ;;; | |
| 368 | ;;; DejaNews functions. | |
| 369 | ;;; | |
| 370 | ||
| 371 | (defun nnweb-dejanews-create-mapping () | |
| 372 | "Perform the search and create an number-to-url alist." | |
| 373 | (save-excursion | |
| 374 | (set-buffer nnweb-buffer) | |
| 375 | (erase-buffer) | |
| 376 | (when (funcall (nnweb-definition 'search) nnweb-search) | |
| 377 | (let ((i 0) | |
| 378 | (more t) | |
| 379 | (case-fold-search t) | |
| 380 | (active (or (cadr (assoc nnweb-group nnweb-group-alist)) | |
| 381 | (cons 1 0))) | |
| 382 | Subject Score Date Newsgroup Author | |
| 383 | map url) | |
| 384 | (while more | |
| 385 | ;; Go through all the article hits on this page. | |
| 386 | (goto-char (point-min)) | |
| 387 | (nnweb-decode-entities) | |
| 388 | (goto-char (point-min)) | |
| 389 | (while (re-search-forward "^ +[0-9]+\\." nil t) | |
| 390 | (narrow-to-region | |
| 391 | (point) | |
| 392 | (cond ((re-search-forward "^ +[0-9]+\\." nil t) | |
| 393 | (match-beginning 0)) | |
| 394 | ((search-forward "\n\n" nil t) | |
| 395 | (point)) | |
| 396 | (t | |
| 397 | (point-max)))) | |
| 398 | (goto-char (point-min)) | |
| 399 | (when (looking-at ".*HREF=\"\\([^\"]+\\)\"") | |
| 400 | (setq url (match-string 1))) | |
| 401 | (nnweb-remove-markup) | |
| 402 | (goto-char (point-min)) | |
| 403 | (while (search-forward "\t" nil t) | |
| 404 | (replace-match " ")) | |
| 405 | (goto-char (point-min)) | |
| 406 | (while (re-search-forward "^ +\\([^:]+\\): +\\(.*\\)$" nil t) | |
| 407 | (set (intern (match-string 1)) (match-string 2))) | |
| 408 | (widen) | |
| 409 | (when (string-match "#[0-9]+/[0-9]+ *$" Subject) | |
| 410 | (setq Subject (substring Subject 0 (match-beginning 0)))) | |
| 411 | (incf i) | |
| 412 | (unless (nnweb-get-hashtb url) | |
| 413 | (push | |
| 414 | (list | |
| 415 | (incf (cdr active)) | |
| 416 | (make-full-mail-header | |
| 417 | (cdr active) (concat "(" Newsgroup ") " Subject) Author Date | |
| 418 | (concat "<" (nnweb-identifier url) "@dejanews>") | |
| 419 | nil 0 (string-to-int Score) url)) | |
| 420 | map) | |
| 421 | (nnweb-set-hashtb (cadar map) (car map)))) | |
| 422 | ;; See whether there is a "Get next 20 hits" button here. | |
| 423 | (if (or (not (re-search-forward | |
| 424 | "HREF=\"\\([^\"]+\\)\">Get next" nil t)) | |
| 425 | (>= i nnweb-max-hits)) | |
| 426 | (setq more nil) | |
| 427 | ;; Yup -- fetch it. | |
| 428 | (setq more (match-string 1)) | |
| 429 | (erase-buffer) | |
| 430 | (url-insert-file-contents more))) | |
| 431 | ;; Return the articles in the right order. | |
| 432 | (setq nnweb-articles | |
| 433 | (sort (nconc nnweb-articles map) | |
| 434 | (lambda (s1 s2) (< (car s1) (car s2))))))))) | |
| 435 | ||
| 436 | (defun nnweb-dejanews-wash-article () | |
| 437 | (let ((case-fold-search t)) | |
| 438 | (goto-char (point-min)) | |
| 439 | (re-search-forward "<PRE>" nil t) | |
| 440 | (delete-region (point-min) (point)) | |
| 441 | (re-search-forward "</PRE>" nil t) | |
| 442 | (delete-region (point) (point-max)) | |
| 443 | (nnweb-remove-markup) | |
| 444 | (goto-char (point-min)) | |
| 445 | (while (and (looking-at " *$") | |
| 446 | (not (eobp))) | |
| 447 | (gnus-delete-line)) | |
| 448 | (while (looking-at "\\(^[^ ]+:\\) *") | |
| 449 | (replace-match "\\1 " t) | |
| 450 | (forward-line 1)) | |
| 451 | (when (re-search-forward "\n\n+" nil t) | |
| a8151ef7 LMI |
452 | (replace-match "\n" t t)) |
| 453 | (goto-char (point-min)) | |
| 454 | (when (search-forward "[More Headers]" nil t) | |
| 455 | (replace-match "" t t)))) | |
| eec82323 LMI |
456 | |
| 457 | (defun nnweb-dejanews-search (search) | |
| 458 | (nnweb-fetch-form | |
| 459 | (nnweb-definition 'address) | |
| 460 | `(("query" . ,search) | |
| 461 | ("defaultOp" . "AND") | |
| 462 | ("svcclass" . "dncurrent") | |
| 463 | ("maxhits" . "100") | |
| 464 | ("format" . "verbose") | |
| 465 | ("threaded" . "0") | |
| 466 | ("showsort" . "score") | |
| 467 | ("agesign" . "1") | |
| 468 | ("ageweight" . "1"))) | |
| 469 | t) | |
| 470 | ||
| 471 | (defun nnweb-dejanews-identity (url) | |
| 472 | "Return an unique identifier based on URL." | |
| 473 | (if (string-match "recnum=\\([0-9]+\\)" url) | |
| 474 | (match-string 1 url) | |
| 475 | url)) | |
| 476 | ||
| 477 | ;;; | |
| 478 | ;;; InReference | |
| 479 | ;;; | |
| 480 | ||
| 481 | (defun nnweb-reference-create-mapping () | |
| 482 | "Perform the search and create an number-to-url alist." | |
| 483 | (save-excursion | |
| 484 | (set-buffer nnweb-buffer) | |
| 485 | (erase-buffer) | |
| 486 | (when (funcall (nnweb-definition 'search) nnweb-search) | |
| 487 | (let ((i 0) | |
| 488 | (more t) | |
| 489 | (case-fold-search t) | |
| 490 | (active (or (cadr (assoc nnweb-group nnweb-group-alist)) | |
| 491 | (cons 1 0))) | |
| 492 | Subject Score Date Newsgroups From Message-ID | |
| 493 | map url) | |
| 494 | (while more | |
| 495 | ;; Go through all the article hits on this page. | |
| 496 | (goto-char (point-min)) | |
| 497 | (search-forward "</pre><hr>" nil t) | |
| 498 | (delete-region (point-min) (point)) | |
| 499 | ;(nnweb-decode-entities) | |
| 500 | (goto-char (point-min)) | |
| 501 | (while (re-search-forward "^ +[0-9]+\\." nil t) | |
| 502 | (narrow-to-region | |
| 503 | (point) | |
| 504 | (if (re-search-forward "^$" nil t) | |
| 505 | (match-beginning 0) | |
| 506 | (point-max))) | |
| 507 | (goto-char (point-min)) | |
| 508 | (when (looking-at ".*href=\"\\([^\"]+\\)\"") | |
| 509 | (setq url (match-string 1))) | |
| 510 | (nnweb-remove-markup) | |
| 511 | (goto-char (point-min)) | |
| 512 | (while (search-forward "\t" nil t) | |
| 513 | (replace-match " ")) | |
| 514 | (goto-char (point-min)) | |
| 515 | (while (re-search-forward "^\\([^:]+\\): \\(.*\\)$" nil t) | |
| 516 | (set (intern (match-string 1)) (match-string 2))) | |
| 517 | (widen) | |
| 518 | (search-forward "</pre>" nil t) | |
| 519 | (incf i) | |
| 520 | (unless (nnweb-get-hashtb url) | |
| 521 | (push | |
| 522 | (list | |
| 523 | (incf (cdr active)) | |
| 524 | (make-full-mail-header | |
| 525 | (cdr active) (concat "(" Newsgroups ") " Subject) From Date | |
| 526 | Message-ID | |
| 527 | nil 0 (string-to-int Score) url)) | |
| 528 | map) | |
| 529 | (nnweb-set-hashtb (cadar map) (car map)))) | |
| 530 | (setq more nil)) | |
| 531 | ;; Return the articles in the right order. | |
| 532 | (setq nnweb-articles | |
| 533 | (sort (nconc nnweb-articles map) | |
| 534 | (lambda (s1 s2) (< (car s1) (car s2))))))))) | |
| 535 | ||
| 536 | (defun nnweb-reference-wash-article () | |
| 537 | (let ((case-fold-search t)) | |
| 538 | (goto-char (point-min)) | |
| 539 | (re-search-forward "^</center><hr>" nil t) | |
| 540 | (delete-region (point-min) (point)) | |
| 541 | (search-forward "<pre>" nil t) | |
| 542 | (forward-line -1) | |
| 543 | (let ((body (point-marker))) | |
| 544 | (search-forward "</pre>" nil t) | |
| 545 | (delete-region (point) (point-max)) | |
| 546 | (nnweb-remove-markup) | |
| 547 | (goto-char (point-min)) | |
| 548 | (while (looking-at " *$") | |
| 549 | (gnus-delete-line)) | |
| 550 | (narrow-to-region (point-min) body) | |
| 551 | (while (and (re-search-forward "^$" nil t) | |
| 552 | (not (eobp))) | |
| 553 | (gnus-delete-line)) | |
| 554 | (goto-char (point-min)) | |
| 555 | (while (looking-at "\\(^[^ ]+:\\) *") | |
| 556 | (replace-match "\\1 " t) | |
| 557 | (forward-line 1)) | |
| 558 | (goto-char (point-min)) | |
| 559 | (when (re-search-forward "^References:" nil t) | |
| 560 | (narrow-to-region | |
| 561 | (point) (if (re-search-forward "^$\\|^[^:]+:" nil t) | |
| 562 | (match-beginning 0) | |
| 563 | (point-max))) | |
| 564 | (goto-char (point-min)) | |
| 565 | (while (not (eobp)) | |
| 566 | (unless (looking-at "References") | |
| 567 | (insert "\t") | |
| 568 | (forward-line 1))) | |
| 569 | (goto-char (point-min)) | |
| 570 | (while (search-forward "," nil t) | |
| 571 | (replace-match " " t t))) | |
| 572 | (widen) | |
| 573 | (set-marker body nil)))) | |
| 574 | ||
| 575 | (defun nnweb-reference-search (search) | |
| a8151ef7 LMI |
576 | (url-insert-file-contents |
| 577 | (concat | |
| 578 | (nnweb-definition 'address) | |
| 579 | "?" | |
| 580 | (nnweb-encode-www-form-urlencoded | |
| 581 | `(("search" . "advanced") | |
| 582 | ("querytext" . ,search) | |
| 583 | ("subj" . "") | |
| 584 | ("name" . "") | |
| 585 | ("login" . "") | |
| 586 | ("host" . "") | |
| 587 | ("organization" . "") | |
| 588 | ("groups" . "") | |
| 589 | ("keywords" . "") | |
| 590 | ("choice" . "Search") | |
| 591 | ("startmonth" . "Jul") | |
| 592 | ("startday" . "25") | |
| 593 | ("startyear" . "1996") | |
| 594 | ("endmonth" . "Aug") | |
| 595 | ("endday" . "24") | |
| 596 | ("endyear" . "1996") | |
| 597 | ("mode" . "Quick") | |
| 598 | ("verbosity" . "Verbose") | |
| 599 | ("ranking" . "Relevance") | |
| 600 | ("first" . "1") | |
| 601 | ("last" . "25") | |
| 602 | ("score" . "50"))))) | |
| 603 | (setq buffer-file-name nil) | |
| eec82323 LMI |
604 | t) |
| 605 | ||
| 606 | ;;; | |
| 607 | ;;; Alta Vista | |
| 608 | ;;; | |
| 609 | ||
| 610 | (defun nnweb-altavista-create-mapping () | |
| 611 | "Perform the search and create an number-to-url alist." | |
| 612 | (save-excursion | |
| 613 | (set-buffer nnweb-buffer) | |
| 614 | (erase-buffer) | |
| 615 | (let ((part 0)) | |
| 616 | (when (funcall (nnweb-definition 'search) nnweb-search part) | |
| 617 | (let ((i 0) | |
| 618 | (more t) | |
| 619 | (case-fold-search t) | |
| 620 | (active (or (cadr (assoc nnweb-group nnweb-group-alist)) | |
| 621 | (cons 1 0))) | |
| 622 | subject date from id group | |
| 623 | map url) | |
| 624 | (while more | |
| 625 | ;; Go through all the article hits on this page. | |
| 626 | (goto-char (point-min)) | |
| 627 | (search-forward "<dt>" nil t) | |
| 628 | (delete-region (point-min) (match-beginning 0)) | |
| 629 | (goto-char (point-min)) | |
| 630 | (while (search-forward "<dt>" nil t) | |
| 631 | (replace-match "\n<blubb>")) | |
| 632 | (nnweb-decode-entities) | |
| 633 | (goto-char (point-min)) | |
| 634 | (while (re-search-forward "<blubb>.*href=\"\\([^\"]+\\)\"><strong>\\([^>]*\\)</strong></a><dd>\\([^-]+\\)- <b>\\([^<]+\\)<.*href=\"news:\\([^\"]+\\)\">.*\">\\(.+\\)</a><P>" | |
| 635 | nil t) | |
| 636 | (setq url (match-string 1) | |
| 637 | subject (match-string 2) | |
| 638 | date (match-string 3) | |
| 639 | group (match-string 4) | |
| 640 | id (concat "<" (match-string 5) ">") | |
| 641 | from (match-string 6)) | |
| 642 | (incf i) | |
| 643 | (unless (nnweb-get-hashtb url) | |
| 644 | (push | |
| 645 | (list | |
| 646 | (incf (cdr active)) | |
| 647 | (make-full-mail-header | |
| 648 | (cdr active) (concat "(" group ") " subject) from date | |
| 649 | id nil 0 0 url)) | |
| 650 | map) | |
| 651 | (nnweb-set-hashtb (cadar map) (car map)))) | |
| 652 | ;; See if we want more. | |
| 653 | (when (or (not nnweb-articles) | |
| 654 | (>= i nnweb-max-hits) | |
| 655 | (not (funcall (nnweb-definition 'search) | |
| 656 | nnweb-search (incf part)))) | |
| 657 | (setq more nil))) | |
| 658 | ;; Return the articles in the right order. | |
| 659 | (setq nnweb-articles | |
| 660 | (sort (nconc nnweb-articles map) | |
| 661 | (lambda (s1 s2) (< (car s1) (car s2)))))))))) | |
| 662 | ||
| 663 | (defun nnweb-altavista-wash-article () | |
| 664 | (goto-char (point-min)) | |
| 665 | (let ((case-fold-search t)) | |
| 666 | (when (re-search-forward "^<strong>" nil t) | |
| 667 | (delete-region (point-min) (match-beginning 0))) | |
| 668 | (goto-char (point-min)) | |
| 669 | (while (looking-at "<strong>\\([^ ]+\\) +</strong> +\\(.*\\)$") | |
| 670 | (replace-match "\\1: \\2" t) | |
| 671 | (forward-line 1)) | |
| 672 | (when (re-search-backward "^References:" nil t) | |
| 673 | (narrow-to-region (point) (progn (forward-line 1) (point))) | |
| 674 | (goto-char (point-min)) | |
| 675 | (while (re-search-forward "<A.*\\?id@\\([^\"]+\\)\">[0-9]+</A>" nil t) | |
| 676 | (replace-match "<\\1> " t))) | |
| 677 | (widen) | |
| 678 | (nnweb-remove-markup))) | |
| 679 | ||
| 680 | (defun nnweb-altavista-search (search &optional part) | |
| a8151ef7 LMI |
681 | (url-insert-file-contents |
| 682 | (concat | |
| 683 | (nnweb-definition 'address) | |
| 684 | "?" | |
| 685 | (nnweb-encode-www-form-urlencoded | |
| 686 | `(("pg" . "aq") | |
| 687 | ("what" . "news") | |
| 688 | ,@(when part `(("stq" . ,(int-to-string (* part 30))))) | |
| 689 | ("fmt" . "d") | |
| 690 | ("q" . ,search) | |
| 691 | ("r" . "") | |
| 692 | ("d0" . "") | |
| 693 | ("d1" . ""))))) | |
| 694 | (setq buffer-file-name nil) | |
| 695 | t) | |
| eec82323 LMI |
696 | |
| 697 | (provide 'nnweb) | |
| 698 | ||
| 699 | ;;; nnweb.el ends here |