Commit | Line | Data |
---|---|---|
8c8b8430 | 1 | ;;; url.el --- Uniform Resource Locator retrieval tool |
42b369cd | 2 | |
acaf905b | 3 | ;; Copyright (C) 1996-1999, 2001, 2004-2012 Free Software Foundation, Inc. |
42b369cd | 4 | |
8c8b8430 | 5 | ;; Author: Bill Perry <wmperry@gnu.org> |
8c8b8430 SM |
6 | ;; Keywords: comm, data, processes, hypermedia |
7 | ||
42b369cd SM |
8 | ;; This file is part of GNU Emacs. |
9 | ;; | |
4936186e | 10 | ;; GNU Emacs is free software: you can redistribute it and/or modify |
42b369cd | 11 | ;; it under the terms of the GNU General Public License as published by |
4936186e GM |
12 | ;; the Free Software Foundation, either version 3 of the License, or |
13 | ;; (at your option) any later version. | |
14 | ||
42b369cd SM |
15 | ;; GNU Emacs is distributed in the hope that it will be useful, |
16 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
17 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
18 | ;; GNU General Public License for more details. | |
4936186e | 19 | |
42b369cd | 20 | ;; You should have received a copy of the GNU General Public License |
4936186e | 21 | ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>. |
42b369cd SM |
22 | |
23 | ;;; Commentary: | |
8c8b8430 SM |
24 | |
25 | ;; Registered URI schemes: http://www.iana.org/assignments/uri-schemes | |
26 | ||
42b369cd SM |
27 | ;;; Code: |
28 | ||
8c8b8430 | 29 | (eval-when-compile (require 'cl)) |
8c8b8430 | 30 | |
aa8f8277 GM |
31 | (require 'mailcap) |
32 | ||
8c8b8430 SM |
33 | (eval-when-compile |
34 | (require 'mm-decode) | |
35 | (require 'mm-view)) | |
36 | ||
8c8b8430 SM |
37 | (require 'url-vars) |
38 | (require 'url-cookie) | |
39 | (require 'url-history) | |
40 | (require 'url-expand) | |
41 | (require 'url-privacy) | |
42 | (require 'url-methods) | |
43 | (require 'url-proxy) | |
44 | (require 'url-parse) | |
45 | (require 'url-util) | |
46 | ||
4577244f | 47 | |
4577244f | 48 | (defcustom url-configuration-directory |
091b0137 | 49 | (locate-user-emacs-file "url/" ".url/") |
4577244f GM |
50 | "Directory used by the URL package for cookies, history, etc." |
51 | :type 'directory | |
52 | :group 'url) | |
8c8b8430 SM |
53 | |
54 | (defun url-do-setup () | |
d1ce47b0 | 55 | "Setup the URL package. |
8c8b8430 SM |
56 | This is to avoid conflict with user settings if URL is dumped with |
57 | Emacs." | |
58 | (unless url-setup-done | |
59 | ||
60 | ;; Make OS/2 happy | |
61 | ;;(push '("http" "80") tcp-binary-process-input-services) | |
62 | ||
63 | (mailcap-parse-mailcaps) | |
64 | (mailcap-parse-mimetypes) | |
71ddfde5 | 65 | |
8c8b8430 SM |
66 | ;; Register all the authentication schemes we can handle |
67 | (url-register-auth-scheme "basic" nil 4) | |
68 | (url-register-auth-scheme "digest" nil 7) | |
69 | ||
70 | (setq url-cookie-file | |
71 | (or url-cookie-file | |
72 | (expand-file-name "cookies" url-configuration-directory))) | |
71ddfde5 | 73 | |
8c8b8430 SM |
74 | (setq url-history-file |
75 | (or url-history-file | |
76 | (expand-file-name "history" url-configuration-directory))) | |
71ddfde5 | 77 | |
8c8b8430 SM |
78 | ;; Parse the global history file if it exists, so that it can be used |
79 | ;; for URL completion, etc. | |
80 | (url-history-parse-history) | |
81 | (url-history-setup-save-timer) | |
82 | ||
83 | ;; Ditto for cookies | |
84 | (url-cookie-setup-save-timer) | |
85 | (url-cookie-parse-file url-cookie-file) | |
86 | ||
87 | ;; Read in proxy gateways | |
88 | (let ((noproxy (and (not (assoc "no_proxy" url-proxy-services)) | |
89 | (or (getenv "NO_PROXY") | |
90 | (getenv "no_PROXY") | |
91 | (getenv "no_proxy"))))) | |
92 | (if noproxy | |
93 | (setq url-proxy-services | |
94 | (cons (cons "no_proxy" | |
95 | (concat "\\(" | |
96 | (mapconcat | |
97 | (lambda (x) | |
98 | (cond | |
99 | ((= x ?,) "\\|") | |
100 | ((= x ? ) "") | |
101 | ((= x ?.) (regexp-quote ".")) | |
102 | ((= x ?*) ".*") | |
103 | ((= x ??) ".") | |
104 | (t (char-to-string x)))) | |
105 | noproxy "") "\\)")) | |
106 | url-proxy-services)))) | |
107 | ||
8c8b8430 SM |
108 | (url-setup-privacy-info) |
109 | (run-hooks 'url-load-hook) | |
110 | (setq url-setup-done t))) | |
111 | ||
112 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; | |
113 | ;;; Retrieval functions | |
114 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; | |
9b4cf231 SM |
115 | |
116 | (defvar url-redirect-buffer nil | |
117 | "New buffer into which the retrieval will take place. | |
118 | Sometimes while retrieving a URL, the URL library needs to use another buffer | |
119 | than the one returned initially by `url-retrieve'. In this case, it sets this | |
120 | variable in the original buffer as a forwarding pointer.") | |
121 | ||
1968bb1b LI |
122 | (defvar url-retrieve-number-of-calls 0) |
123 | (autoload 'url-cache-prune-cache "url-cache") | |
124 | ||
2ef88a69 | 125 | ;;;###autoload |
08b8ba9f | 126 | (defun url-retrieve (url callback &optional cbargs silent) |
8c8b8430 | 127 | "Retrieve URL asynchronously and call CALLBACK with CBARGS when finished. |
497803ed RS |
128 | URL is either a string or a parsed URL. |
129 | ||
130 | CALLBACK is called when the object has been completely retrieved, with | |
8c8b8430 | 131 | the current buffer containing the object, and any MIME headers associated |
1b244f61 CY |
132 | with it. It is called as (apply CALLBACK STATUS CBARGS). |
133 | STATUS is a list with an even number of elements representing | |
134 | what happened during the request, with most recent events first, | |
135 | or an empty list if no events have occurred. Each pair is one of: | |
5695d1dd CY |
136 | |
137 | \(:redirect REDIRECTED-TO) - the request was redirected to this URL | |
138 | \(:error (ERROR-SYMBOL . DATA)) - an error occurred. The error can be | |
139 | signaled with (signal ERROR-SYMBOL DATA). | |
8c8b8430 SM |
140 | |
141 | Return the buffer URL will load into, or nil if the process has | |
5695d1dd CY |
142 | already completed (i.e. URL was a mailto URL or similar; in this case |
143 | the callback is not called). | |
144 | ||
145 | The variables `url-request-data', `url-request-method' and | |
146 | `url-request-extra-headers' can be dynamically bound around the | |
147 | request; dynamic binding of other variables doesn't necessarily | |
08b8ba9f LMI |
148 | take effect. |
149 | ||
150 | If SILENT, then don't message progress reports and the like." | |
5695d1dd CY |
151 | ;;; XXX: There is code in Emacs that does dynamic binding |
152 | ;;; of the following variables around url-retrieve: | |
153 | ;;; url-standalone-mode, url-gateway-unplugged, w3-honor-stylesheets, | |
154 | ;;; url-confirmation-func, url-cookie-multiple-line, | |
155 | ;;; url-cookie-{{,secure-}storage,confirmation} | |
156 | ;;; url-standalone-mode and url-gateway-unplugged should work as | |
157 | ;;; usual. url-confirmation-func is only used in nnwarchive.el and | |
158 | ;;; webmail.el; the latter should be updated. Is | |
159 | ;;; url-cookie-multiple-line needed anymore? The other url-cookie-* | |
160 | ;;; are (for now) only used in synchronous retrievals. | |
08b8ba9f | 161 | (url-retrieve-internal url callback (cons nil cbargs) silent)) |
5695d1dd | 162 | |
08b8ba9f | 163 | (defun url-retrieve-internal (url callback cbargs &optional silent) |
5695d1dd CY |
164 | "Internal function; external interface is `url-retrieve'. |
165 | CBARGS is what the callback will actually receive - the first item is | |
08b8ba9f LMI |
166 | the list of events, as described in the docstring of `url-retrieve'. |
167 | ||
168 | If SILENT, don't message progress reports and the like." | |
8c8b8430 SM |
169 | (url-do-setup) |
170 | (url-gc-dead-buffers) | |
171 | (if (stringp url) | |
172 | (set-text-properties 0 (length url) nil url)) | |
173 | (if (not (vectorp url)) | |
174 | (setq url (url-generic-parse-url url))) | |
175 | (if (not (functionp callback)) | |
176 | (error "Must provide a callback function to url-retrieve")) | |
177 | (unless (url-type url) | |
178 | (error "Bad url: %s" (url-recreate-url url))) | |
08b8ba9f | 179 | (setf (url-silent url) silent) |
1968bb1b LI |
180 | ;; Once in a while, remove old entries from the URL cache. |
181 | (when (zerop (% url-retrieve-number-of-calls 1000)) | |
182 | (url-cache-prune-cache)) | |
183 | (setq url-retrieve-number-of-calls (1+ url-retrieve-number-of-calls)) | |
8c8b8430 SM |
184 | (let ((loader (url-scheme-get-property (url-type url) 'loader)) |
185 | (url-using-proxy (if (url-host url) | |
186 | (url-find-proxy-for-url url (url-host url)))) | |
187 | (buffer nil) | |
188 | (asynch (url-scheme-get-property (url-type url) 'asynchronous-p))) | |
189 | (if url-using-proxy | |
190 | (setq asynch t | |
191 | loader 'url-proxy)) | |
192 | (if asynch | |
08b8ba9f LMI |
193 | (let ((url-current-object url)) |
194 | (setq buffer (funcall loader url callback cbargs))) | |
8c8b8430 SM |
195 | (setq buffer (funcall loader url)) |
196 | (if buffer | |
42b369cd | 197 | (with-current-buffer buffer |
8c8b8430 | 198 | (apply callback cbargs)))) |
11b5750f RS |
199 | (if url-history-track |
200 | (url-history-update-url url (current-time))) | |
8c8b8430 SM |
201 | buffer)) |
202 | ||
2ef88a69 | 203 | ;;;###autoload |
8c8b8430 SM |
204 | (defun url-retrieve-synchronously (url) |
205 | "Retrieve URL synchronously. | |
206 | Return the buffer containing the data, or nil if there are no data | |
207 | associated with it (the case for dired, info, or mailto URLs that need | |
208 | no further processing). URL is either a string or a parsed URL." | |
209 | (url-do-setup) | |
210 | ||
211 | (lexical-let ((retrieval-done nil) | |
212 | (asynch-buffer nil)) | |
213 | (setq asynch-buffer | |
214 | (url-retrieve url (lambda (&rest ignored) | |
215 | (url-debug 'retrieval "Synchronous fetching done (%S)" (current-buffer)) | |
216 | (setq retrieval-done t | |
217 | asynch-buffer (current-buffer))))) | |
7f954571 SM |
218 | (if (null asynch-buffer) |
219 | ;; We do not need to do anything, it was a mailto or something | |
220 | ;; similar that takes processing completely outside of the URL | |
221 | ;; package. | |
222 | nil | |
223 | (let ((proc (get-buffer-process asynch-buffer))) | |
224 | ;; If the access method was synchronous, `retrieval-done' should | |
225 | ;; hopefully already be set to t. If it is nil, and `proc' is also | |
226 | ;; nil, it implies that the async process is not running in | |
227 | ;; asynch-buffer. This happens e.g. for FTP files. In such a case | |
228 | ;; url-file.el should probably set something like a `url-process' | |
229 | ;; buffer-local variable so we can find the exact process that we | |
230 | ;; should be waiting for. In the mean time, we'll just wait for any | |
231 | ;; process output. | |
944b2ab6 SM |
232 | (while (not retrieval-done) |
233 | (url-debug 'retrieval | |
234 | "Spinning in url-retrieve-synchronously: %S (%S)" | |
235 | retrieval-done asynch-buffer) | |
9b4cf231 SM |
236 | (if (buffer-local-value 'url-redirect-buffer asynch-buffer) |
237 | (setq proc (get-buffer-process | |
238 | (setq asynch-buffer | |
239 | (buffer-local-value 'url-redirect-buffer | |
240 | asynch-buffer)))) | |
241 | (if (and proc (memq (process-status proc) | |
242 | '(closed exit signal failed)) | |
243 | ;; Make sure another process hasn't been started. | |
244 | (eq proc (or (get-buffer-process asynch-buffer) proc))) | |
245 | ;; FIXME: It's not clear whether url-retrieve's callback is | |
246 | ;; guaranteed to be called or not. It seems that url-http | |
247 | ;; decides sometimes consciously not to call it, so it's not | |
248 | ;; clear that it's a bug, but even then we need to decide how | |
249 | ;; url-http can then warn us that the download has completed. | |
250 | ;; In the mean time, we use this here workaround. | |
5695d1dd CY |
251 | ;; XXX: The callback must always be called. Any |
252 | ;; exception is a bug that should be fixed, not worked | |
253 | ;; around. | |
9ffb9521 RS |
254 | (progn ;; Call delete-process so we run any sentinel now. |
255 | (delete-process proc) | |
256 | (setq retrieval-done t))) | |
799fba8f SM |
257 | ;; We used to use `sit-for' here, but in some cases it wouldn't |
258 | ;; work because apparently pending keyboard input would always | |
259 | ;; interrupt it before it got a chance to handle process input. | |
260 | ;; `sleep-for' was tried but it lead to other forms of | |
261 | ;; hanging. --Stef | |
da6062e6 | 262 | (unless (or (with-local-quit |
947612be MH |
263 | (accept-process-output proc)) |
264 | (null proc)) | |
799fba8f | 265 | ;; accept-process-output returned nil, maybe because the process |
947612be MH |
266 | ;; exited (and may have been replaced with another). If we got |
267 | ;; a quit, just stop. | |
268 | (when quit-flag | |
269 | (delete-process proc)) | |
270 | (setq proc (and (not quit-flag) | |
271 | (get-buffer-process asynch-buffer))))))) | |
8c8b8430 SM |
272 | asynch-buffer))) |
273 | ||
274 | (defun url-mm-callback (&rest ignored) | |
275 | (let ((handle (mm-dissect-buffer t))) | |
7f954571 SM |
276 | (url-mark-buffer-as-dead (current-buffer)) |
277 | (with-current-buffer | |
278 | (generate-new-buffer (url-recreate-url url-current-object)) | |
8c8b8430 SM |
279 | (if (eq (mm-display-part handle) 'external) |
280 | (progn | |
281 | (set-process-sentinel | |
282 | ;; Fixme: this shouldn't have to know the form of the | |
283 | ;; undisplayer produced by `mm-display-part'. | |
284 | (get-buffer-process (cdr (mm-handle-undisplayer handle))) | |
285 | `(lambda (proc event) | |
286 | (mm-destroy-parts (quote ,handle)))) | |
287 | (message "Viewing externally") | |
288 | (kill-buffer (current-buffer))) | |
289 | (display-buffer (current-buffer)) | |
71ddfde5 | 290 | (add-hook 'kill-buffer-hook |
14e25c87 MY |
291 | `(lambda () (mm-destroy-parts ',handle)) |
292 | nil | |
293 | t))))) | |
8c8b8430 SM |
294 | |
295 | (defun url-mm-url (url) | |
296 | "Retrieve URL and pass to the appropriate viewing application." | |
717c6bde SM |
297 | ;; These requires could advantageously be moved to url-mm-callback or |
298 | ;; turned into autoloads, but I suspect that it would introduce some bugs | |
299 | ;; because loading those files from a process sentinel or filter may | |
da6062e6 | 300 | ;; result in some undesirable corner cases. |
8c8b8430 SM |
301 | (require 'mm-decode) |
302 | (require 'mm-view) | |
303 | (url-retrieve url 'url-mm-callback nil)) | |
304 | ||
305 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; | |
306 | ;;; Miscellaneous | |
307 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; | |
308 | (defvar url-dead-buffer-list nil) | |
309 | ||
310 | (defun url-mark-buffer-as-dead (buff) | |
311 | (push buff url-dead-buffer-list)) | |
312 | ||
313 | (defun url-gc-dead-buffers () | |
314 | (let ((buff)) | |
315 | (while (setq buff (pop url-dead-buffer-list)) | |
316 | (if (buffer-live-p buff) | |
317 | (kill-buffer buff))))) | |
318 | ||
319 | (cond | |
320 | ((fboundp 'display-warning) | |
321 | (defalias 'url-warn 'display-warning)) | |
322 | ((fboundp 'warn) | |
323 | (defun url-warn (class message &optional level) | |
324 | (warn "(%s/%s) %s" class (or level 'warning) message))) | |
325 | (t | |
326 | (defun url-warn (class message &optional level) | |
42b369cd | 327 | (with-current-buffer (get-buffer-create "*URL-WARNINGS*") |
8c8b8430 SM |
328 | (goto-char (point-max)) |
329 | (save-excursion | |
330 | (insert (format "(%s/%s) %s\n" class (or level 'warning) message))) | |
331 | (display-buffer (current-buffer)))))) | |
332 | ||
333 | (provide 'url) | |
334 | ||
335 | ;;; url.el ends here |