Commit | Line | Data |
---|---|---|
5c77c3ed LMI |
1 | ;;; url-queue.el --- Fetching web pages in parallel |
2 | ||
3 | ;; Copyright (C) 2011 Free Software Foundation, Inc. | |
4 | ||
5 | ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org> | |
6 | ;; Keywords: comm | |
7 | ||
8 | ;; This file is part of GNU Emacs. | |
9 | ||
10 | ;; GNU Emacs is free software: you can redistribute it and/or modify | |
11 | ;; it under the terms of the GNU General Public License as published by | |
12 | ;; the Free Software Foundation, either version 3 of the License, or | |
13 | ;; (at your option) any later version. | |
14 | ||
15 | ;; GNU Emacs is distributed in the hope that it will be useful, | |
16 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
17 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
18 | ;; GNU General Public License for more details. | |
19 | ||
20 | ;; You should have received a copy of the GNU General Public License | |
21 | ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>. | |
22 | ||
23 | ;;; Commentary: | |
24 | ||
25 | ;; The point of this package is to allow fetching web pages in | |
26 | ;; parallel -- but control the level of parallelism to avoid DoS-ing | |
27 | ;; web servers and Emacs. | |
28 | ||
29 | ;;; Code: | |
30 | ||
31 | (eval-when-compile (require 'cl)) | |
32 | (require 'browse-url) | |
33 | ||
34 | (defcustom url-queue-parallel-processes 4 | |
35 | "The number of concurrent processes." | |
36 | :type 'integer | |
37 | :group 'url) | |
38 | ||
39 | (defcustom url-queue-timeout 5 | |
40 | "How long to let a job live once it's started (in seconds)." | |
41 | :type 'integer | |
42 | :group 'url) | |
43 | ||
44 | ;;; Internal variables. | |
45 | ||
46 | (defvar url-queue nil) | |
47 | ||
48 | (defstruct url-queue | |
49 | url callback cbargs silentp | |
50 | process start-time) | |
51 | ||
52 | (defun url-queue-retrieve (url callback &optional cbargs silent) | |
53 | "Retrieve URL asynchronously and call CALLBACK with CBARGS when finished. | |
54 | Like `url-retrieve' (which see for details of the arguments), but | |
55 | controls the level of parallelism via the | |
56 | `url-queue-parallel-processes' variable." | |
57 | (setq url-queue | |
58 | (append url-queue | |
59 | (list (make-url-queue :url url | |
60 | :callback callback | |
61 | :cbargs cbargs | |
62 | :silentp silent)))) | |
63 | (url-queue-run-queue)) | |
64 | ||
65 | (defun url-queue-run-queue () | |
66 | (url-queue-prune-old-entries) | |
67 | (let ((running 0) | |
68 | waiting) | |
69 | (dolist (entry url-queue) | |
08da93f1 LMI |
70 | (cond |
71 | ((url-queue-start-time entry) | |
72 | (incf running)) | |
73 | ((not waiting) | |
74 | (setq waiting entry)))) | |
5c77c3ed LMI |
75 | (when (and waiting |
76 | (< running url-queue-parallel-processes)) | |
77 | (setf (url-queue-start-time waiting) (float-time)) | |
78 | (url-queue-start-retrieve waiting)))) | |
79 | ||
80 | (defun url-queue-callback-function (status job) | |
81 | (setq url-queue (delq job url-queue)) | |
82 | (url-queue-run-queue) | |
83 | (apply (url-queue-callback job) (cons status (url-queue-cbargs job)))) | |
84 | ||
85 | (defun url-queue-start-retrieve (job) | |
86 | (setf (url-queue-process job) | |
87 | (ignore-errors | |
88 | (url-retrieve (url-queue-url job) | |
89 | #'url-queue-callback-function (list job) | |
90 | (url-queue-silentp job))))) | |
91 | ||
92 | (defun url-queue-prune-old-entries () | |
93 | (let (dead-jobs) | |
94 | (dolist (job url-queue) | |
95 | ;; Kill jobs that have lasted longer than five seconds. | |
96 | (when (and (url-queue-start-time job) | |
97 | (> (- (float-time) (url-queue-start-time job)) | |
98 | url-queue-timeout)) | |
99 | (push job dead-jobs))) | |
100 | (dolist (job dead-jobs) | |
101 | (when (processp (url-queue-process job)) | |
102 | (ignore-errors | |
103 | (delete-process (url-queue-process job))) | |
104 | (ignore-errors | |
105 | (kill-buffer (process-buffer (url-queue-process job)))) | |
106 | (setq url-queue (delq job url-queue)))))) | |
107 | ||
108 | (provide 'url-queue) | |
109 | ||
110 | ;;; url-queue.el ends here |