lisp/url/url-util.el

   1 ;;; url-util.el --- Miscellaneous helper routines for URL library
   2
   3 ;; Copyright (C) 1996, 1997, 1998, 1999, 2001, 2004,
   4 ;;   2005, 2006, 2007, 2008 Free Software Foundation, Inc.
   5
   6 ;; Author: Bill Perry <wmperry@gnu.org>
   7 ;; Keywords: comm, data, processes
   8
   9 ;; This file is part of GNU Emacs.
  10 ;;
  11 ;; GNU Emacs is free software; you can redistribute it and/or modify
  12 ;; it under the terms of the GNU General Public License as published by
  13 ;; the Free Software Foundation; either version 3, or (at your option)
  14 ;; any later version.
  15 ;;
  16 ;; GNU Emacs is distributed in the hope that it will be useful,
  17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;; GNU General Public License for more details.
  20 ;;
  21 ;; You should have received a copy of the GNU General Public License
  22 ;; along with GNU Emacs; see the file COPYING.  If not, write to the
  23 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  24 ;; Boston, MA 02110-1301, USA.
  25
  26 ;;; Commentary:
  27
  28 ;;; Code:
  29
  30 (require 'url-parse)
  31 (eval-when-compile (require 'cl))
  32 (autoload 'timezone-parse-date "timezone")
  33 (autoload 'timezone-make-date-arpa-standard "timezone")
  34 (autoload 'mail-header-extract "mailheader")
  35
  36 (defvar url-parse-args-syntax-table
  37   (copy-syntax-table emacs-lisp-mode-syntax-table)
  38   "A syntax table for parsing sgml attributes.")
  39
  40 (modify-syntax-entry ?' "\"" url-parse-args-syntax-table)
  41 (modify-syntax-entry ?` "\"" url-parse-args-syntax-table)
  42 (modify-syntax-entry ?{ "(" url-parse-args-syntax-table)
  43 (modify-syntax-entry ?} ")" url-parse-args-syntax-table)
  44
  45 ;;;###autoload
  46 (defcustom url-debug nil
  47   "*What types of debug messages from the URL library to show.
  48 Debug messages are logged to the *URL-DEBUG* buffer.
  49
  50 If t, all messages will be logged.
  51 If a number, all messages will be logged, as well shown via `message'.
  52 If a list, it is a list of the types of messages to be logged."
  53   :type '(choice (const :tag "none" nil)
  54                  (const :tag "all" t)
  55                  (checklist :tag "custom"
  56                             (const :tag "HTTP" :value http)
  57                             (const :tag "DAV" :value dav)
  58                             (const :tag "General" :value retrieval)
  59                             (const :tag "Filename handlers" :value handlers)
  60                             (symbol :tag "Other")))
  61   :group 'url-hairy)
  62
  63 ;;;###autoload
  64 (defun url-debug (tag &rest args)
  65   (if quit-flag
  66       (error "Interrupted!"))
  67   (if (or (eq url-debug t)
  68           (numberp url-debug)
  69           (and (listp url-debug) (memq tag url-debug)))
  70       (with-current-buffer (get-buffer-create "*URL-DEBUG*")
  71         (goto-char (point-max))
  72         (insert (symbol-name tag) " -> " (apply 'format args) "\n")
  73         (if (numberp url-debug)
  74             (apply 'message args)))))
  75
  76 ;;;###autoload
  77 (defun url-parse-args (str &optional nodowncase)
  78   ;; Return an assoc list of attribute/value pairs from an RFC822-type string
  79   (let (
  80         name                            ; From name=
  81         value                           ; its value
  82         results                         ; Assoc list of results
  83         name-pos                        ; Start of XXXX= position
  84         val-pos                         ; Start of value position
  85         st
  86         nd
  87         )
  88     (save-excursion
  89       (save-restriction
  90         (set-buffer (get-buffer-create " *urlparse-temp*"))
  91         (set-syntax-table url-parse-args-syntax-table)
  92         (erase-buffer)
  93         (insert str)
  94         (setq st (point-min)
  95               nd (point-max))
  96         (set-syntax-table url-parse-args-syntax-table)
  97         (narrow-to-region st nd)
  98         (goto-char (point-min))
  99         (while (not (eobp))
 100           (skip-chars-forward "; \n\t")
 101           (setq name-pos (point))
 102           (skip-chars-forward "^ \n\t=;")
 103           (if (not nodowncase)
 104               (downcase-region name-pos (point)))
 105           (setq name (buffer-substring name-pos (point)))
 106           (skip-chars-forward " \t\n")
 107           (if (/= (or (char-after (point)) 0)  ?=) ; There is no value
 108               (setq value nil)
 109             (skip-chars-forward " \t\n=")
 110             (setq val-pos (point)
 111                   value
 112                   (cond
 113                    ((or (= (or (char-after val-pos) 0) ?\")
 114                         (= (or (char-after val-pos) 0) ?'))
 115                     (buffer-substring (1+ val-pos)
 116                                       (condition-case ()
 117                                           (prog2
 118                                               (forward-sexp 1)
 119                                               (1- (point))
 120                                             (skip-chars-forward "\""))
 121                                         (error
 122                                          (skip-chars-forward "^ \t\n")
 123                                          (point)))))
 124                    (t
 125                     (buffer-substring val-pos
 126                                       (progn
 127                                         (skip-chars-forward "^;")
 128                                         (skip-chars-backward " \t")
 129                                         (point)))))))
 130           (setq results (cons (cons name value) results))
 131           (skip-chars-forward "; \n\t"))
 132         results))))
 133
 134 ;;;###autoload
 135 (defun url-insert-entities-in-string (string)
 136   "Convert HTML markup-start characters to entity references in STRING.
 137 Also replaces the \" character, so that the result may be safely used as
 138   an attribute value in a tag.  Returns a new string with the result of the
 139   conversion.  Replaces these characters as follows:
 140     &  ==>  &amp;
 141     <  ==>  &lt;
 142     >  ==>  &gt;
 143     \"  ==>  &quot;"
 144   (if (string-match "[&<>\"]" string)
 145       (save-excursion
 146         (set-buffer (get-buffer-create " *entity*"))
 147         (erase-buffer)
 148         (buffer-disable-undo (current-buffer))
 149         (insert string)
 150         (goto-char (point-min))
 151         (while (progn
 152                  (skip-chars-forward "^&<>\"")
 153                  (not (eobp)))
 154           (insert (cdr (assq (char-after (point))
 155                              '((?\" . "&quot;")
 156                                (?& . "&amp;")
 157                                (?< . "&lt;")
 158                                (?> . "&gt;")))))
 159           (delete-char 1))
 160         (buffer-string))
 161     string))
 162
 163 ;;;###autoload
 164 (defun url-normalize-url (url)
 165   "Return a 'normalized' version of URL.
 166 Strips out default port numbers, etc."
 167   (let (type data retval)
 168     (setq data (url-generic-parse-url url)
 169           type (url-type data))
 170     (if (member type '("www" "about" "mailto" "info"))
 171         (setq retval url)
 172       (setf (url-target data) nil)
 173       (setq retval (url-recreate-url data)))
 174     retval))
 175
 176 ;;;###autoload
 177 (defun url-lazy-message (&rest args)
 178   "Just like `message', but is a no-op if called more than once a second.
 179 Will not do anything if `url-show-status' is nil."
 180   (if (or (null url-show-status)
 181           (active-minibuffer-window)
 182           (= url-lazy-message-time
 183              (setq url-lazy-message-time (nth 1 (current-time)))))
 184       nil
 185     (apply 'message args)))
 186
 187 ;;;###autoload
 188 (defun url-get-normalized-date (&optional specified-time)
 189  "Return a 'real' date string that most HTTP servers can understand."
 190  (let ((system-time-locale "C"))
 191   (format-time-string "%a, %d %b %Y %T GMT"
 192    (or specified-time (current-time)) t)))
 193
 194 ;;;###autoload
 195 (defun url-eat-trailing-space (x)
 196   "Remove spaces/tabs at the end of a string."
 197   (let ((y (1- (length x)))
 198         (skip-chars (list ?  ?\t ?\n)))
 199     (while (and (>= y 0) (memq (aref x y) skip-chars))
 200       (setq y (1- y)))
 201     (substring x 0 (1+ y))))
 202
 203 ;;;###autoload
 204 (defun url-strip-leading-spaces (x)
 205   "Remove spaces at the front of a string."
 206   (let ((y (1- (length x)))
 207         (z 0)
 208         (skip-chars (list ?  ?\t ?\n)))
 209     (while (and (<= z y) (memq (aref x z) skip-chars))
 210       (setq z (1+ z)))
 211     (substring x z nil)))
 212
 213 ;;;###autoload
 214 (defun url-pretty-length (n)
 215   (cond
 216    ((< n 1024)
 217     (format "%d bytes" n))
 218    ((< n (* 1024 1024))
 219     (format "%dk" (/ n 1024.0)))
 220    (t
 221     (format "%2.2fM" (/ n (* 1024 1024.0))))))
 222
 223 ;;;###autoload
 224 (defun url-display-percentage (fmt perc &rest args)
 225   (when url-show-status
 226     (if (null fmt)
 227         (if (fboundp 'clear-progress-display)
 228             (clear-progress-display))
 229       (if (and (fboundp 'progress-display) perc)
 230           (apply 'progress-display fmt perc args)
 231         (apply 'message fmt args)))))
 232
 233 ;;;###autoload
 234 (defun url-percentage (x y)
 235   (if (fboundp 'float)
 236       (round (* 100 (/ x (float y))))
 237     (/ (* x 100) y)))
 238
 239 ;;;###autoload
 240 (defun url-file-directory (file)
 241   "Return the directory part of FILE, for a URL."
 242   (cond
 243    ((null file) "")
 244    ((string-match (eval-when-compile (regexp-quote "?")) file)
 245     (file-name-directory (substring file 0 (match-beginning 0))))
 246    (t (file-name-directory file))))
 247
 248 ;;;###autoload
 249 (defun url-file-nondirectory (file)
 250   "Return the nondirectory part of FILE, for a URL."
 251   (cond
 252    ((null file) "")
 253    ((string-match (eval-when-compile (regexp-quote "?")) file)
 254     (file-name-nondirectory (substring file 0 (match-beginning 0))))
 255    (t (file-name-nondirectory file))))
 256
 257 ;;;###autoload
 258 (defun url-parse-query-string (query &optional downcase allow-newlines)
 259   (let (retval pairs cur key val)
 260     (setq pairs (split-string query "&"))
 261     (while pairs
 262       (setq cur (car pairs)
 263             pairs (cdr pairs))
 264       (if (not (string-match "=" cur))
 265           nil                           ; Grace
 266         (setq key (url-unhex-string (substring cur 0 (match-beginning 0))
 267                                     allow-newlines))
 268         (setq val (url-unhex-string (substring cur (match-end 0) nil)
 269                                     allow-newlines))
 270         (if downcase
 271             (setq key (downcase key)))
 272         (setq cur (assoc key retval))
 273         (if cur
 274             (setcdr cur (cons val (cdr cur)))
 275           (setq retval (cons (list key val) retval)))))
 276     retval))
 277
 278 (defun url-unhex (x)
 279   (if (> x ?9)
 280       (if (>= x ?a)
 281           (+ 10 (- x ?a))
 282         (+ 10 (- x ?A)))
 283     (- x ?0)))
 284
 285 ;; Fixme: Is this definition better, and does it ever matter?
 286
 287 ;; (defun url-unhex-string (str &optional allow-newlines)
 288 ;;   "Remove %XX, embedded spaces, etc in a url.
 289 ;; If optional second argument ALLOW-NEWLINES is non-nil, then allow the
 290 ;; decoding of carriage returns and line feeds in the string, which is normally
 291 ;; forbidden in URL encoding."
 292 ;;   (setq str (or str ""))
 293 ;;   (setq str (replace-regexp-in-string "%[[:xdigit:]]\\{2\\}"
 294 ;;                                    (lambda (match)
 295 ;;                                      (string (string-to-number
 296 ;;                                               (substring match 1) 16)))
 297 ;;                                    str t t))
 298 ;;   (if allow-newlines
 299 ;;       (replace-regexp-in-string "[\n\r]" (lambda (match)
 300 ;;                                         (format "%%%.2X" (aref match 0)))
 301 ;;                              str t t)
 302 ;;     str))
 303
 304 ;;;###autoload
 305 (defun url-unhex-string (str &optional allow-newlines)
 306   "Remove %XX embedded spaces, etc in a url.
 307 If optional second argument ALLOW-NEWLINES is non-nil, then allow the
 308 decoding of carriage returns and line feeds in the string, which is normally
 309 forbidden in URL encoding."
 310   (setq str (or str ""))
 311   (let ((tmp "")
 312         (case-fold-search t))
 313     (while (string-match "%[0-9a-f][0-9a-f]" str)
 314       (let* ((start (match-beginning 0))
 315              (ch1 (url-unhex (elt str (+ start 1))))
 316              (code (+ (* 16 ch1)
 317                       (url-unhex (elt str (+ start 2))))))
 318         (setq tmp (concat
 319                    tmp (substring str 0 start)
 320                    (cond
 321                     (allow-newlines
 322                      (char-to-string code))
 323                     ((or (= code ?\n) (= code ?\r))
 324                      " ")
 325                     (t (char-to-string code))))
 326               str (substring str (match-end 0)))))
 327     (setq tmp (concat tmp str))
 328     tmp))
 329
 330 (defconst url-unreserved-chars
 331   '(
 332     ?a ?b ?c ?d ?e ?f ?g ?h ?i ?j ?k ?l ?m ?n ?o ?p ?q ?r ?s ?t ?u ?v ?w ?x ?y ?z
 333     ?A ?B ?C ?D ?E ?F ?G ?H ?I ?J ?K ?L ?M ?N ?O ?P ?Q ?R ?S ?T ?U ?V ?W ?X ?Y ?Z
 334     ?0 ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9
 335     ?- ?_ ?. ?! ?~ ?* ?' ?\( ?\))
 336   "A list of characters that are _NOT_ reserved in the URL spec.
 337 This is taken from RFC 2396.")
 338
 339 ;;;###autoload
 340 (defun url-hexify-string (string)
 341   "Return a new string that is STRING URI-encoded.
 342 First, STRING is converted to utf-8, if necessary.  Then, for each
 343 character in the utf-8 string, those found in `url-unreserved-chars'
 344 are left as-is, all others are represented as a three-character
 345 string: \"%\" followed by two lowercase hex digits."
 346   ;; To go faster and avoid a lot of consing, we could do:
 347   ;;
 348   ;; (defconst url-hexify-table
 349   ;;   (let ((map (make-vector 256 nil)))
 350   ;;     (dotimes (byte 256) (aset map byte
 351   ;;                               (if (memq byte url-unreserved-chars)
 352   ;;                                   (char-to-string byte)
 353   ;;                                 (format "%%%02x" byte))))
 354   ;;     map))
 355   ;;
 356   ;; (mapconcat (curry 'aref url-hexify-table) ...)
 357   (mapconcat (lambda (byte)
 358                (if (memq byte url-unreserved-chars)
 359                    (char-to-string byte)
 360                  (format "%%%02x" byte)))
 361              (if (multibyte-string-p string)
 362                  (encode-coding-string string 'utf-8)
 363                string)
 364              ""))
 365
 366 ;;;###autoload
 367 (defun url-file-extension (fname &optional x)
 368   "Return the filename extension of FNAME.
 369 If optional variable X is t,
 370 then return the basename of the file with the extension stripped off."
 371   (if (and fname
 372            (setq fname (url-file-nondirectory fname))
 373            (string-match "\\.[^./]+$" fname))
 374       (if x (substring fname 0 (match-beginning 0))
 375         (substring fname (match-beginning 0) nil))
 376     ;;
 377     ;; If fname has no extension, and x then return fname itself instead of
 378     ;; nothing. When caching it allows the correct .hdr file to be produced
 379     ;; for filenames without extension.
 380     ;;
 381     (if x
 382         fname
 383       "")))
 384
 385 ;;;###autoload
 386 (defun url-truncate-url-for-viewing (url &optional width)
 387   "Return a shortened version of URL that is WIDTH characters or less wide.
 388 WIDTH defaults to the current frame width."
 389   (let* ((fr-width (or width (frame-width)))
 390          (str-width (length url))
 391          (fname nil)
 392          (modified 0)
 393          (urlobj nil))
 394     ;; The first thing that can go are the search strings
 395     (if (and (>= str-width fr-width)
 396              (string-match "?" url))
 397         (setq url (concat (substring url 0 (match-beginning 0)) "?...")
 398               str-width (length url)))
 399     (if (< str-width fr-width)
 400         nil                             ; Hey, we are done!
 401       (setq urlobj (url-generic-parse-url url)
 402             fname (url-filename urlobj)
 403             fr-width (- fr-width 4))
 404       (while (and (>= str-width fr-width)
 405                   (string-match "/" fname))
 406         (setq fname (substring fname (match-end 0) nil)
 407               modified (1+ modified))
 408         (setf (url-filename urlobj) fname)
 409         (setq url (url-recreate-url urlobj)
 410               str-width (length url)))
 411       (if (> modified 1)
 412           (setq fname (concat "/.../" fname))
 413         (setq fname (concat "/" fname)))
 414       (setf (url-filename urlobj) fname)
 415       (setq url (url-recreate-url urlobj)))
 416     url))
 417
 418 ;;;###autoload
 419 (defun url-view-url (&optional no-show)
 420   "View the current document's URL.
 421 Optional argument NO-SHOW means just return the URL, don't show it in
 422 the minibuffer.
 423
 424 This uses `url-current-object', set locally to the buffer."
 425   (interactive)
 426   (if (not url-current-object)
 427       nil
 428     (if no-show
 429         (url-recreate-url url-current-object)
 430       (message "%s" (url-recreate-url url-current-object)))))
 431
 432 (eval-and-compile
 433   (defvar url-get-url-filename-chars "-%.?@a-zA-Z0-9()_/:~=&"
 434     "Valid characters in a URL")
 435   )
 436
 437 (defun url-get-url-at-point (&optional pt)
 438   "Get the URL closest to point, but don't change position.
 439 Has a preference for looking backward when not directly on a symbol."
 440   ;; Not at all perfect - point must be right in the name.
 441   (save-excursion
 442     (if pt (goto-char pt))
 443     (let (start url)
 444       (save-excursion
 445         ;; first see if you're just past a filename
 446         (if (not (eobp))
 447             (if (looking-at "[] \t\n[{}()]") ; whitespace or some parens
 448                 (progn
 449                   (skip-chars-backward " \n\t\r({[]})")
 450                   (if (not (bobp))
 451                       (backward-char 1)))))
 452         (if (and (char-after (point))
 453                  (string-match (eval-when-compile
 454                                  (concat "[" url-get-url-filename-chars "]"))
 455                                (char-to-string (char-after (point)))))
 456             (progn
 457               (skip-chars-backward url-get-url-filename-chars)
 458               (setq start (point))
 459               (skip-chars-forward url-get-url-filename-chars))
 460           (setq start (point)))
 461         (setq url (buffer-substring-no-properties start (point))))
 462       (if (and url (string-match "^(.*)\\.?$" url))
 463           (setq url (match-string 1 url)))
 464       (if (and url (string-match "^URL:" url))
 465           (setq url (substring url 4 nil)))
 466       (if (and url (string-match "\\.$" url))
 467           (setq url (substring url 0 -1)))
 468       (if (and url (string-match "^www\\." url))
 469           (setq url (concat "http://" url)))
 470       (if (and url (not (string-match url-nonrelative-link url)))
 471           (setq url nil))
 472       url)))
 473
 474 (defun url-generate-unique-filename (&optional fmt)
 475   "Generate a unique filename in `url-temporary-directory'."
 476   (if (not fmt)
 477       (let ((base (format "url-tmp.%d" (user-real-uid)))
 478             (fname "")
 479             (x 0))
 480         (setq fname (format "%s%d" base x))
 481         (while (file-exists-p
 482                 (expand-file-name fname url-temporary-directory))
 483           (setq x (1+ x)
 484                 fname (concat base (int-to-string x))))
 485         (expand-file-name fname url-temporary-directory))
 486     (let ((base (concat "url" (int-to-string (user-real-uid))))
 487           (fname "")
 488           (x 0))
 489       (setq fname (format fmt (concat base (int-to-string x))))
 490       (while (file-exists-p
 491               (expand-file-name fname url-temporary-directory))
 492         (setq x (1+ x)
 493               fname (format fmt (concat base (int-to-string x)))))
 494       (expand-file-name fname url-temporary-directory))))
 495
 496 (defun url-extract-mime-headers ()
 497   "Set `url-current-mime-headers' in current buffer."
 498   (save-excursion
 499     (goto-char (point-min))
 500     (unless url-current-mime-headers
 501       (set (make-local-variable 'url-current-mime-headers)
 502            (mail-header-extract)))))
 503
 504 (defun url-make-private-file (file)
 505   "Make FILE only readable and writable by the current user.
 506 Creates FILE and its parent directories if they do not exist."
 507   (let ((dir (file-name-directory file)))
 508     (when dir
 509       ;; For historical reasons.
 510       (make-directory dir t)))
 511   ;; Based on doc-view-make-safe-dir.
 512   (condition-case nil
 513       (let ((umask (default-file-modes)))
 514         (unwind-protect
 515             (progn
 516               (set-default-file-modes #o0600)
 517               (with-temp-buffer
 518                 (write-region (point-min) (point-max)
 519                               file nil 'silent nil 'excl)))
 520           (set-default-file-modes umask)))
 521     (file-already-exists
 522      (if (file-symlink-p file)
 523          (error "Danger: `%s' is a symbolic link" file))
 524      (set-file-modes file #o0600))))
 525
 526 (provide 'url-util)
 527
 528 ;; arch-tag: 24352abc-5a5a-412e-90cd-313b26bed5c9
 529 ;;; url-util.el ends here