Merge from emacs-24; up to 2012-04-22T13:58:00Z!cyd@gnu.org
[bpt/emacs.git] / lisp / url / url-util.el
1 ;;; url-util.el --- Miscellaneous helper routines for URL library
2
3 ;; Copyright (C) 1996-1999, 2001, 2004-2012 Free Software Foundation, Inc.
4
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Keywords: comm, data, processes
7
8 ;; This file is part of GNU Emacs.
9 ;;
10 ;; GNU Emacs is free software: you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation, either version 3 of the License, or
13 ;; (at your option) any later version.
14
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
19
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
22
23 ;;; Commentary:
24
25 ;;; Code:
26
27 (require 'url-parse)
28 (require 'url-vars)
29 (eval-when-compile (require 'cl))
30 (autoload 'timezone-parse-date "timezone")
31 (autoload 'timezone-make-date-arpa-standard "timezone")
32 (autoload 'mail-header-extract "mailheader")
33
34 (defvar url-parse-args-syntax-table
35 (copy-syntax-table emacs-lisp-mode-syntax-table)
36 "A syntax table for parsing sgml attributes.")
37
38 (modify-syntax-entry ?' "\"" url-parse-args-syntax-table)
39 (modify-syntax-entry ?` "\"" url-parse-args-syntax-table)
40 (modify-syntax-entry ?{ "(" url-parse-args-syntax-table)
41 (modify-syntax-entry ?} ")" url-parse-args-syntax-table)
42
43 ;;;###autoload
44 (defcustom url-debug nil
45 "What types of debug messages from the URL library to show.
46 Debug messages are logged to the *URL-DEBUG* buffer.
47
48 If t, all messages will be logged.
49 If a number, all messages will be logged, as well shown via `message'.
50 If a list, it is a list of the types of messages to be logged."
51 :type '(choice (const :tag "none" nil)
52 (const :tag "all" t)
53 (checklist :tag "custom"
54 (const :tag "HTTP" :value http)
55 (const :tag "DAV" :value dav)
56 (const :tag "General" :value retrieval)
57 (const :tag "Filename handlers" :value handlers)
58 (symbol :tag "Other")))
59 :group 'url-hairy)
60
61 ;;;###autoload
62 (defun url-debug (tag &rest args)
63 (if quit-flag
64 (error "Interrupted!"))
65 (if (or (eq url-debug t)
66 (numberp url-debug)
67 (and (listp url-debug) (memq tag url-debug)))
68 (with-current-buffer (get-buffer-create "*URL-DEBUG*")
69 (goto-char (point-max))
70 (insert (symbol-name tag) " -> " (apply 'format args) "\n")
71 (if (numberp url-debug)
72 (apply 'message args)))))
73
74 ;;;###autoload
75 (defun url-parse-args (str &optional nodowncase)
76 ;; Return an assoc list of attribute/value pairs from an RFC822-type string
77 (let (
78 name ; From name=
79 value ; its value
80 results ; Assoc list of results
81 name-pos ; Start of XXXX= position
82 val-pos ; Start of value position
83 st
84 nd
85 )
86 (save-excursion
87 (save-restriction
88 (set-buffer (get-buffer-create " *urlparse-temp*"))
89 (set-syntax-table url-parse-args-syntax-table)
90 (erase-buffer)
91 (insert str)
92 (setq st (point-min)
93 nd (point-max))
94 (set-syntax-table url-parse-args-syntax-table)
95 (narrow-to-region st nd)
96 (goto-char (point-min))
97 (while (not (eobp))
98 (skip-chars-forward "; \n\t")
99 (setq name-pos (point))
100 (skip-chars-forward "^ \n\t=;")
101 (if (not nodowncase)
102 (downcase-region name-pos (point)))
103 (setq name (buffer-substring name-pos (point)))
104 (skip-chars-forward " \t\n")
105 (if (/= (or (char-after (point)) 0) ?=) ; There is no value
106 (setq value nil)
107 (skip-chars-forward " \t\n=")
108 (setq val-pos (point)
109 value
110 (cond
111 ((or (= (or (char-after val-pos) 0) ?\")
112 (= (or (char-after val-pos) 0) ?'))
113 (buffer-substring (1+ val-pos)
114 (condition-case ()
115 (prog2
116 (forward-sexp 1)
117 (1- (point))
118 (skip-chars-forward "\""))
119 (error
120 (skip-chars-forward "^ \t\n")
121 (point)))))
122 (t
123 (buffer-substring val-pos
124 (progn
125 (skip-chars-forward "^;")
126 (skip-chars-backward " \t")
127 (point)))))))
128 (setq results (cons (cons name value) results))
129 (skip-chars-forward "; \n\t"))
130 results))))
131
132 ;;;###autoload
133 (defun url-insert-entities-in-string (string)
134 "Convert HTML markup-start characters to entity references in STRING.
135 Also replaces the \" character, so that the result may be safely used as
136 an attribute value in a tag. Returns a new string with the result of the
137 conversion. Replaces these characters as follows:
138 & ==> &amp;
139 < ==> &lt;
140 > ==> &gt;
141 \" ==> &quot;"
142 (if (string-match "[&<>\"]" string)
143 (with-current-buffer (get-buffer-create " *entity*")
144 (erase-buffer)
145 (buffer-disable-undo (current-buffer))
146 (insert string)
147 (goto-char (point-min))
148 (while (progn
149 (skip-chars-forward "^&<>\"")
150 (not (eobp)))
151 (insert (cdr (assq (char-after (point))
152 '((?\" . "&quot;")
153 (?& . "&amp;")
154 (?< . "&lt;")
155 (?> . "&gt;")))))
156 (delete-char 1))
157 (buffer-string))
158 string))
159
160 ;;;###autoload
161 (defun url-normalize-url (url)
162 "Return a 'normalized' version of URL.
163 Strips out default port numbers, etc."
164 (let (type data retval)
165 (setq data (url-generic-parse-url url)
166 type (url-type data))
167 (if (member type '("www" "about" "mailto" "info"))
168 (setq retval url)
169 ;; FIXME all this does, and all this function seems to do in
170 ;; most cases, is remove any trailing "#anchor" part of a url.
171 (setf (url-target data) nil)
172 (setq retval (url-recreate-url data)))
173 retval))
174
175 ;;;###autoload
176 (defun url-lazy-message (&rest args)
177 "Just like `message', but is a no-op if called more than once a second.
178 Will not do anything if `url-show-status' is nil."
179 (if (or (and url-current-object
180 (url-silent url-current-object))
181 (null url-show-status)
182 (active-minibuffer-window)
183 (= url-lazy-message-time
184 (setq url-lazy-message-time (nth 1 (current-time)))))
185 nil
186 (apply 'message args)))
187
188 ;;;###autoload
189 (defun url-get-normalized-date (&optional specified-time)
190 "Return a 'real' date string that most HTTP servers can understand."
191 (let ((system-time-locale "C"))
192 (format-time-string "%a, %d %b %Y %T GMT"
193 (or specified-time (current-time)) t)))
194
195 ;;;###autoload
196 (defun url-eat-trailing-space (x)
197 "Remove spaces/tabs at the end of a string."
198 (let ((y (1- (length x)))
199 (skip-chars (list ? ?\t ?\n)))
200 (while (and (>= y 0) (memq (aref x y) skip-chars))
201 (setq y (1- y)))
202 (substring x 0 (1+ y))))
203
204 ;;;###autoload
205 (defun url-strip-leading-spaces (x)
206 "Remove spaces at the front of a string."
207 (let ((y (1- (length x)))
208 (z 0)
209 (skip-chars (list ? ?\t ?\n)))
210 (while (and (<= z y) (memq (aref x z) skip-chars))
211 (setq z (1+ z)))
212 (substring x z nil)))
213
214 ;;;###autoload
215 (defun url-pretty-length (n)
216 (cond
217 ((< n 1024)
218 (format "%d bytes" n))
219 ((< n (* 1024 1024))
220 (format "%dk" (/ n 1024.0)))
221 (t
222 (format "%2.2fM" (/ n (* 1024 1024.0))))))
223
224 ;;;###autoload
225 (defun url-display-percentage (fmt perc &rest args)
226 (when (and url-show-status
227 (or (null url-current-object)
228 (not (url-silent url-current-object))))
229 (if (null fmt)
230 (if (fboundp 'clear-progress-display)
231 (clear-progress-display))
232 (if (and (fboundp 'progress-display) perc)
233 (apply 'progress-display fmt perc args)
234 (apply 'message fmt args)))))
235
236 ;;;###autoload
237 (defun url-percentage (x y)
238 (if (fboundp 'float)
239 (round (* 100 (/ x (float y))))
240 (/ (* x 100) y)))
241
242 ;;;###autoload
243 (defalias 'url-basepath 'url-file-directory)
244
245 ;;;###autoload
246 (defun url-file-directory (file)
247 "Return the directory part of FILE, for a URL."
248 (cond
249 ((null file) "")
250 ((string-match "\\?" file)
251 (file-name-directory (substring file 0 (match-beginning 0))))
252 (t (file-name-directory file))))
253
254 ;;;###autoload
255 (defun url-file-nondirectory (file)
256 "Return the nondirectory part of FILE, for a URL."
257 (cond
258 ((null file) "")
259 ((string-match "\\?" file)
260 (file-name-nondirectory (substring file 0 (match-beginning 0))))
261 (t (file-name-nondirectory file))))
262
263 ;;;###autoload
264 (defun url-parse-query-string (query &optional downcase allow-newlines)
265 (let (retval pairs cur key val)
266 (setq pairs (split-string query "&"))
267 (while pairs
268 (setq cur (car pairs)
269 pairs (cdr pairs))
270 (if (not (string-match "=" cur))
271 nil ; Grace
272 (setq key (url-unhex-string (substring cur 0 (match-beginning 0))
273 allow-newlines))
274 (setq val (url-unhex-string (substring cur (match-end 0) nil)
275 allow-newlines))
276 (if downcase
277 (setq key (downcase key)))
278 (setq cur (assoc key retval))
279 (if cur
280 (setcdr cur (cons val (cdr cur)))
281 (setq retval (cons (list key val) retval)))))
282 retval))
283
284 (defun url-unhex (x)
285 (if (> x ?9)
286 (if (>= x ?a)
287 (+ 10 (- x ?a))
288 (+ 10 (- x ?A)))
289 (- x ?0)))
290
291 ;; Fixme: Is this definition better, and does it ever matter?
292
293 ;; (defun url-unhex-string (str &optional allow-newlines)
294 ;; "Remove %XX, embedded spaces, etc in a url.
295 ;; If optional second argument ALLOW-NEWLINES is non-nil, then allow the
296 ;; decoding of carriage returns and line feeds in the string, which is normally
297 ;; forbidden in URL encoding."
298 ;; (setq str (or str ""))
299 ;; (setq str (replace-regexp-in-string "%[[:xdigit:]]\\{2\\}"
300 ;; (lambda (match)
301 ;; (string (string-to-number
302 ;; (substring match 1) 16)))
303 ;; str t t))
304 ;; (if allow-newlines
305 ;; (replace-regexp-in-string "[\n\r]" (lambda (match)
306 ;; (format "%%%.2X" (aref match 0)))
307 ;; str t t)
308 ;; str))
309
310 ;;;###autoload
311 (defun url-unhex-string (str &optional allow-newlines)
312 "Remove %XX embedded spaces, etc in a URL.
313 If optional second argument ALLOW-NEWLINES is non-nil, then allow the
314 decoding of carriage returns and line feeds in the string, which is normally
315 forbidden in URL encoding."
316 (setq str (or str ""))
317 (let ((tmp "")
318 (case-fold-search t))
319 (while (string-match "%[0-9a-f][0-9a-f]" str)
320 (let* ((start (match-beginning 0))
321 (ch1 (url-unhex (elt str (+ start 1))))
322 (code (+ (* 16 ch1)
323 (url-unhex (elt str (+ start 2))))))
324 (setq tmp (concat
325 tmp (substring str 0 start)
326 (cond
327 (allow-newlines
328 (byte-to-string code))
329 ((or (= code ?\n) (= code ?\r))
330 " ")
331 (t (byte-to-string code))))
332 str (substring str (match-end 0)))))
333 (concat tmp str)))
334
335 (defconst url-unreserved-chars
336 '(?a ?b ?c ?d ?e ?f ?g ?h ?i ?j ?k ?l ?m ?n ?o ?p ?q ?r ?s ?t ?u ?v ?w ?x ?y ?z
337 ?A ?B ?C ?D ?E ?F ?G ?H ?I ?J ?K ?L ?M ?N ?O ?P ?Q ?R ?S ?T ?U ?V ?W ?X ?Y ?Z
338 ?0 ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9
339 ?- ?_ ?. ?~)
340 "List of characters that are unreserved in the URL spec.
341 This is taken from RFC 3986 (section 2.3).")
342
343 (defconst url-encoding-table
344 (let ((vec (make-vector 256 nil)))
345 (dotimes (byte 256)
346 ;; RFC 3986 (Section 2.1): For consistency, URI producers and
347 ;; normalizers should use uppercase hexadecimal digits for all
348 ;; percent-encodings.
349 (aset vec byte (format "%%%02X" byte)))
350 vec)
351 "Vector translating bytes to URI-encoded %-sequences.")
352
353 (defun url--allowed-chars (char-list)
354 "Return an \"allowed character\" mask (a 256-slot vector).
355 The Nth element is non-nil if character N is in CHAR-LIST. The
356 result can be passed as the second arg to `url-hexify-string'."
357 (let ((vec (make-vector 256 nil)))
358 (dolist (byte char-list)
359 (ignore-errors (aset vec byte t)))
360 vec))
361
362 ;;;###autoload
363 (defun url-hexify-string (string &optional allowed-chars)
364 "URI-encode STRING and return the result.
365 If STRING is multibyte, it is first converted to a utf-8 byte
366 string. Each byte corresponding to an allowed character is left
367 as-is, while all other bytes are converted to a three-character
368 string: \"%\" followed by two upper-case hex digits.
369
370 The allowed characters are specified by ALLOWED-CHARS. If this
371 argument is nil, the list `url-unreserved-chars' determines the
372 allowed characters. Otherwise, ALLOWED-CHARS should be a vector
373 whose Nth element is non-nil if character N is allowed."
374 (unless allowed-chars
375 (setq allowed-chars (url--allowed-chars url-unreserved-chars)))
376 (mapconcat (lambda (byte)
377 (if (aref allowed-chars byte)
378 (char-to-string byte)
379 (aref url-encoding-table byte)))
380 (if (multibyte-string-p string)
381 (encode-coding-string string 'utf-8)
382 string)
383 ""))
384
385 (defconst url-host-allowed-chars
386 ;; Allow % to avoid re-encoding %-encoded sequences.
387 (url--allowed-chars (append '(?% ?! ?$ ?& ?' ?\( ?\) ?* ?+ ?, ?\; ?=)
388 url-unreserved-chars))
389 "Allowed-character byte mask for the host segment of a URI.
390 These characters are specified in RFC 3986, Appendix A.")
391
392 (defconst url-path-allowed-chars
393 (let ((vec (copy-sequence url-host-allowed-chars)))
394 (aset vec ?/ t)
395 (aset vec ?: t)
396 (aset vec ?@ t)
397 vec)
398 "Allowed-character byte mask for the path segment of a URI.
399 These characters are specified in RFC 3986, Appendix A.")
400
401 (defconst url-query-allowed-chars
402 (let ((vec (copy-sequence url-path-allowed-chars)))
403 (aset vec ?? t)
404 vec)
405 "Allowed-character byte mask for the query segment of a URI.
406 These characters are specified in RFC 3986, Appendix A.")
407
408 ;;;###autoload
409 (defun url-encode-url (url)
410 "Return a properly URI-encoded version of URL.
411 This function also performs URI normalization, e.g. converting
412 the scheme to lowercase if it is uppercase. Apart from
413 normalization, if URL is already URI-encoded, this function
414 should return it unchanged."
415 (if (multibyte-string-p url)
416 (setq url (encode-coding-string url 'utf-8)))
417 (let* ((obj (url-generic-parse-url url))
418 (user (url-user obj))
419 (pass (url-password obj))
420 (host (url-host obj))
421 (path-and-query (url-path-and-query obj))
422 (path (car path-and-query))
423 (query (cdr path-and-query))
424 (frag (url-target obj)))
425 (if user
426 (setf (url-user obj) (url-hexify-string user)))
427 (if pass
428 (setf (url-password obj) (url-hexify-string pass)))
429 ;; No special encoding for IPv6 literals.
430 (and host
431 (not (string-match "\\`\\[.*\\]\\'" host))
432 (setf (url-host obj)
433 (url-hexify-string host url-host-allowed-chars)))
434
435 (if path
436 (setq path (url-hexify-string path url-path-allowed-chars)))
437 (if query
438 (setq query (url-hexify-string query url-query-allowed-chars)))
439 (setf (url-filename obj) (if query (concat path "?" query) path))
440
441 (if frag
442 (setf (url-target obj)
443 (url-hexify-string frag url-query-allowed-chars)))
444 (url-recreate-url obj)))
445
446 ;;;###autoload
447 (defun url-file-extension (fname &optional x)
448 "Return the filename extension of FNAME.
449 If optional argument X is t, then return the basename
450 of the file with the extension stripped off."
451 (if (and fname
452 (setq fname (url-file-nondirectory fname))
453 (string-match "\\.[^./]+$" fname))
454 (if x (substring fname 0 (match-beginning 0))
455 (substring fname (match-beginning 0) nil))
456 ;;
457 ;; If fname has no extension, and x then return fname itself instead of
458 ;; nothing. When caching it allows the correct .hdr file to be produced
459 ;; for filenames without extension.
460 ;;
461 (if x
462 fname
463 "")))
464
465 ;;;###autoload
466 (defun url-truncate-url-for-viewing (url &optional width)
467 "Return a shortened version of URL that is WIDTH characters wide or less.
468 WIDTH defaults to the current frame width."
469 (let* ((fr-width (or width (frame-width)))
470 (str-width (length url))
471 (fname nil)
472 (modified 0)
473 (urlobj nil))
474 ;; The first thing that can go are the search strings
475 (if (and (>= str-width fr-width)
476 (string-match "?" url))
477 (setq url (concat (substring url 0 (match-beginning 0)) "?...")
478 str-width (length url)))
479 (if (< str-width fr-width)
480 nil ; Hey, we are done!
481 (setq urlobj (url-generic-parse-url url)
482 fname (url-filename urlobj)
483 fr-width (- fr-width 4))
484 (while (and (>= str-width fr-width)
485 (string-match "/" fname))
486 (setq fname (substring fname (match-end 0) nil)
487 modified (1+ modified))
488 (setf (url-filename urlobj) fname)
489 (setq url (url-recreate-url urlobj)
490 str-width (length url)))
491 (if (> modified 1)
492 (setq fname (concat "/.../" fname))
493 (setq fname (concat "/" fname)))
494 (setf (url-filename urlobj) fname)
495 (setq url (url-recreate-url urlobj)))
496 url))
497
498 ;;;###autoload
499 (defun url-view-url (&optional no-show)
500 "View the current document's URL.
501 Optional argument NO-SHOW means just return the URL, don't show it in
502 the minibuffer.
503
504 This uses `url-current-object', set locally to the buffer."
505 (interactive)
506 (if (not url-current-object)
507 nil
508 (if no-show
509 (url-recreate-url url-current-object)
510 (message "%s" (url-recreate-url url-current-object)))))
511
512 (defvar url-get-url-filename-chars "-%.?@a-zA-Z0-9()_/:~=&"
513 "Valid characters in a URL.")
514
515 (defun url-get-url-at-point (&optional pt)
516 "Get the URL closest to point, but don't change position.
517 Has a preference for looking backward when not directly on a symbol."
518 ;; Not at all perfect - point must be right in the name.
519 (save-excursion
520 (if pt (goto-char pt))
521 (let (start url)
522 (save-excursion
523 ;; first see if you're just past a filename
524 (if (not (eobp))
525 (if (looking-at "[] \t\n[{}()]") ; whitespace or some parens
526 (progn
527 (skip-chars-backward " \n\t\r({[]})")
528 (if (not (bobp))
529 (backward-char 1)))))
530 (if (and (char-after (point))
531 (string-match (concat "[" url-get-url-filename-chars "]")
532 (char-to-string (char-after (point)))))
533 (progn
534 (skip-chars-backward url-get-url-filename-chars)
535 (setq start (point))
536 (skip-chars-forward url-get-url-filename-chars))
537 (setq start (point)))
538 (setq url (buffer-substring-no-properties start (point))))
539 (if (and url (string-match "^(.*)\\.?$" url))
540 (setq url (match-string 1 url)))
541 (if (and url (string-match "^URL:" url))
542 (setq url (substring url 4 nil)))
543 (if (and url (string-match "\\.$" url))
544 (setq url (substring url 0 -1)))
545 (if (and url (string-match "^www\\." url))
546 (setq url (concat "http://" url)))
547 (if (and url (not (string-match url-nonrelative-link url)))
548 (setq url nil))
549 url)))
550
551 (defun url-generate-unique-filename (&optional fmt)
552 "Generate a unique filename in `url-temporary-directory'."
553 ;; This variable is obsolete, but so is this function.
554 (let ((tempdir (with-no-warnings url-temporary-directory)))
555 (if (not fmt)
556 (let ((base (format "url-tmp.%d" (user-real-uid)))
557 (fname "")
558 (x 0))
559 (setq fname (format "%s%d" base x))
560 (while (file-exists-p
561 (expand-file-name fname tempdir))
562 (setq x (1+ x)
563 fname (concat base (int-to-string x))))
564 (expand-file-name fname tempdir))
565 (let ((base (concat "url" (int-to-string (user-real-uid))))
566 (fname "")
567 (x 0))
568 (setq fname (format fmt (concat base (int-to-string x))))
569 (while (file-exists-p
570 (expand-file-name fname tempdir))
571 (setq x (1+ x)
572 fname (format fmt (concat base (int-to-string x)))))
573 (expand-file-name fname tempdir)))))
574 (make-obsolete 'url-generate-unique-filename 'make-temp-file "23.1")
575
576 (defun url-extract-mime-headers ()
577 "Set `url-current-mime-headers' in current buffer."
578 (save-excursion
579 (goto-char (point-min))
580 (unless url-current-mime-headers
581 (set (make-local-variable 'url-current-mime-headers)
582 (mail-header-extract)))))
583
584 (defun url-make-private-file (file)
585 "Make FILE only readable and writable by the current user.
586 Creates FILE and its parent directories if they do not exist."
587 (let ((dir (file-name-directory file)))
588 (when dir
589 ;; For historical reasons.
590 (make-directory dir t)))
591 ;; Based on doc-view-make-safe-dir.
592 (condition-case nil
593 (let ((umask (default-file-modes)))
594 (unwind-protect
595 (progn
596 (set-default-file-modes #o0600)
597 (with-temp-buffer
598 (write-region (point-min) (point-max)
599 file nil 'silent nil 'excl)))
600 (set-default-file-modes umask)))
601 (file-already-exists
602 (if (file-symlink-p file)
603 (error "Danger: `%s' is a symbolic link" file))
604 (set-file-modes file #o0600))))
605
606 (provide 'url-util)
607
608 ;;; url-util.el ends here