source file: /System/Library/Frameworks/Python.framework/Versions/2.3/lib/python2.3/email/Charset.py
file stats: 111 lines, 43 executed: 38.7% covered
   1. # Copyright (C) 2001,2002 Python Software Foundation
   2. # Author: che@debian.org (Ben Gertzfield), barry@zope.com (Barry Warsaw)
   3. 
   4. from types import UnicodeType
   5. from email.Encoders import encode_7or8bit
   6. import email.base64MIME
   7. import email.quopriMIME
   8. 
   9. def _isunicode(s):
  10.     return isinstance(s, UnicodeType)
  11. 
  12. # Python 2.2.1 and beyond has these symbols
  13. try:
  14.     True, False
  15. except NameError:
  16.     True = 1
  17.     False = 0
  18. 
  19. 
  20. 
  21. # Flags for types of header encodings
  22. QP     = 1   # Quoted-Printable
  23. BASE64 = 2   # Base64
  24. SHORTEST = 3 # the shorter of QP and base64, but only for headers
  25. 
  26. # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
  27. MISC_LEN = 7
  28. 
  29. DEFAULT_CHARSET = 'us-ascii'
  30. 
  31. 
  32. 
  33. # Defaults
  34. CHARSETS = {
  35.     # input        header enc  body enc output conv
  36.     'iso-8859-1':  (QP,        QP,      None),
  37.     'iso-8859-2':  (QP,        QP,      None),
  38.     'iso-8859-3':  (QP,        QP,      None),
  39.     'iso-8859-4':  (QP,        QP,      None),
  40.     # iso-8859-5 is Cyrillic, and not especially used
  41.     # iso-8859-6 is Arabic, also not particularly used
  42.     # iso-8859-7 is Greek, QP will not make it readable
  43.     # iso-8859-8 is Hebrew, QP will not make it readable
  44.     'iso-8859-9':  (QP,        QP,      None),
  45.     'iso-8859-10': (QP,        QP,      None),
  46.     # iso-8859-11 is Thai, QP will not make it readable
  47.     'iso-8859-13': (QP,        QP,      None),
  48.     'iso-8859-14': (QP,        QP,      None),
  49.     'iso-8859-15': (QP,        QP,      None),
  50.     'windows-1252':(QP,        QP,      None),
  51.     'viscii':      (QP,        QP,      None),
  52.     'us-ascii':    (None,      None,    None),
  53.     'big5':        (BASE64,    BASE64,  None),
  54.     'gb2312':      (BASE64,    BASE64,  None),
  55.     'euc-jp':      (BASE64,    None,    'iso-2022-jp'),
  56.     'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
  57.     'iso-2022-jp': (BASE64,    None,    None),
  58.     'koi8-r':      (BASE64,    BASE64,  None),
  59.     'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
  60.     # We're making this one up to represent raw unencoded 8-bit
  61.     '8bit':        (None,      BASE64, 'utf-8'),
  62.     }
  63. 
  64. # Aliases for other commonly-used names for character sets.  Map
  65. # them to the real ones used in email.
  66. ALIASES = {
  67.     'latin_1': 'iso-8859-1',
  68.     'latin-1': 'iso-8859-1',
  69.     'latin_2': 'iso-8859-2',
  70.     'latin-2': 'iso-8859-2',
  71.     'latin_3': 'iso-8859-3',
  72.     'latin-3': 'iso-8859-3',
  73.     'latin_4': 'iso-8859-4',
  74.     'latin-4': 'iso-8859-4',
  75.     'latin_5': 'iso-8859-9',
  76.     'latin-5': 'iso-8859-9',
  77.     'latin_6': 'iso-8859-10',
  78.     'latin-6': 'iso-8859-10',
  79.     'latin_7': 'iso-8859-13',
  80.     'latin-7': 'iso-8859-13',
  81.     'latin_8': 'iso-8859-14',
  82.     'latin-8': 'iso-8859-14',
  83.     'latin_9': 'iso-8859-15',
  84.     'latin-9': 'iso-8859-15',
  85.     'cp949':   'ks_c_5601-1987',
  86.     'euc_jp':  'euc-jp',
  87.     'euc_kr':  'euc-kr',
  88.     'ascii':   'us-ascii',
  89.     }
  90. 
  91. # Map charsets to their Unicode codec strings.  Note that Python doesn't come
  92. # with any Asian codecs by default.  Here's where to get them:
  93. #
  94. # Japanese -- http://www.asahi-net.or.jp/~rd6t-kjym/python
  95. # Korean   -- http://sf.net/projects/koco
  96. # Chinese  -- http://sf.net/projects/python-codecs
  97. #
  98. # Note that these codecs have their own lifecycle and may be in varying states
  99. # of stability and useability.
 100. 
 101. CODEC_MAP = {
 102.     'euc-jp':      'japanese.euc-jp',
 103.     'iso-2022-jp': 'japanese.iso-2022-jp',
 104.     'shift_jis':   'japanese.shift_jis',
 105.     'euc-kr':      'korean.euc-kr',
 106.     'ks_c_5601-1987': 'korean.cp949',
 107.     'iso-2022-kr': 'korean.iso-2022-kr',
 108.     'johab':       'korean.johab',
 109.     'gb2132':      'eucgb2312_cn',
 110.     'big5':        'big5_tw',
 111.     'utf-8':       'utf-8',
 112.     # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
 113.     # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
 114.     # Let that stuff pass through without conversion to/from Unicode.
 115.     'us-ascii':    None,
 116.     }
 117. 
 118. 
 119. 
 120. # Convenience functions for extending the above mappings
 121. def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
 122.     """Add character set properties to the global registry.
 123. 
 124.     charset is the input character set, and must be the canonical name of a
 125.     character set.
 126. 
 127.     Optional header_enc and body_enc is either Charset.QP for
 128.     quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
 129.     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
 130.     is only valid for header_enc.  It describes how message headers and
 131.     message bodies in the input charset are to be encoded.  Default is no
 132.     encoding.
 133. 
 134.     Optional output_charset is the character set that the output should be
 135.     in.  Conversions will proceed from input charset, to Unicode, to the
 136.     output charset when the method Charset.convert() is called.  The default
 137.     is to output in the same character set as the input.
 138. 
 139.     Both input_charset and output_charset must have Unicode codec entries in
 140.     the module's charset-to-codec mapping; use add_codec(charset, codecname)
 141.     to add codecs the module does not know about.  See the codecs module's
 142.     documentation for more information.
 143.     """
 144.     if body_enc == SHORTEST:
 145.         raise ValueError, 'SHORTEST not allowed for body_enc'
 146.     CHARSETS[charset] = (header_enc, body_enc, output_charset)
 147. 
 148. 
 149. def add_alias(alias, canonical):
 150.     """Add a character set alias.
 151. 
 152.     alias is the alias name, e.g. latin-1
 153.     canonical is the character set's canonical name, e.g. iso-8859-1
 154.     """
 155.     ALIASES[alias] = canonical
 156. 
 157. 
 158. def add_codec(charset, codecname):
 159.     """Add a codec that map characters in the given charset to/from Unicode.
 160. 
 161.     charset is the canonical name of a character set.  codecname is the name
 162.     of a Python codec, as appropriate for the second argument to the unicode()
 163.     built-in, or to the encode() method of a Unicode string.
 164.     """
 165.     CODEC_MAP[charset] = codecname
 166. 
 167. 
 168. 
 169. class Charset:
 170.     """Map character sets to their email properties.
 171. 
 172.     This class provides information about the requirements imposed on email
 173.     for a specific character set.  It also provides convenience routines for
 174.     converting between character sets, given the availability of the
 175.     applicable codecs.  Given a character set, it will do its best to provide
 176.     information on how to use that character set in an email in an
 177.     RFC-compliant way.
 178. 
 179.     Certain character sets must be encoded with quoted-printable or base64
 180.     when used in email headers or bodies.  Certain character sets must be
 181.     converted outright, and are not allowed in email.  Instances of this
 182.     module expose the following information about a character set:
 183. 
 184.     input_charset: The initial character set specified.  Common aliases
 185.                    are converted to their `official' email names (e.g. latin_1
 186.                    is converted to iso-8859-1).  Defaults to 7-bit us-ascii.
 187. 
 188.     header_encoding: If the character set must be encoded before it can be
 189.                      used in an email header, this attribute will be set to
 190.                      Charset.QP (for quoted-printable), Charset.BASE64 (for
 191.                      base64 encoding), or Charset.SHORTEST for the shortest of
 192.                      QP or BASE64 encoding.  Otherwise, it will be None.
 193. 
 194.     body_encoding: Same as header_encoding, but describes the encoding for the
 195.                    mail message's body, which indeed may be different than the
 196.                    header encoding.  Charset.SHORTEST is not allowed for
 197.                    body_encoding.
 198. 
 199.     output_charset: Some character sets must be converted before the can be
 200.                     used in email headers or bodies.  If the input_charset is
 201.                     one of them, this attribute will contain the name of the
 202.                     charset output will be converted to.  Otherwise, it will
 203.                     be None.
 204. 
 205.     input_codec: The name of the Python codec used to convert the
 206.                  input_charset to Unicode.  If no conversion codec is
 207.                  necessary, this attribute will be None.
 208. 
 209.     output_codec: The name of the Python codec used to convert Unicode
 210.                   to the output_charset.  If no conversion codec is necessary,
 211.                   this attribute will have the same value as the input_codec.
 212.     """
 213.     def __init__(self, input_charset=DEFAULT_CHARSET):
 214.         # RFC 2046, $4.1.2 says charsets are not case sensitive
 215.         input_charset = input_charset.lower()
 216.         # Set the input charset after filtering through the aliases
 217.         self.input_charset = ALIASES.get(input_charset, input_charset)
 218.         # We can try to guess which encoding and conversion to use by the
 219.         # charset_map dictionary.  Try that first, but let the user override
 220.         # it.
 221.         henc, benc, conv = CHARSETS.get(self.input_charset,
 222.                                         (SHORTEST, BASE64, None))
 223.         # Set the attributes, allowing the arguments to override the default.
 224.         self.header_encoding = henc
 225.         self.body_encoding = benc
 226.         self.output_charset = ALIASES.get(conv, conv)
 227.         # Now set the codecs.  If one isn't defined for input_charset,
 228.         # guess and try a Unicode codec with the same name as input_codec.
 229.         self.input_codec = CODEC_MAP.get(self.input_charset,
 230.                                          self.input_charset)
 231.         self.output_codec = CODEC_MAP.get(self.output_charset,
 232.                                             self.input_codec)
 233. 
 234.     def __str__(self):
 235.         return self.input_charset.lower()
 236. 
 237.     __repr__ = __str__
 238. 
 239.     def __eq__(self, other):
 240.         return str(self) == str(other).lower()
 241. 
 242.     def __ne__(self, other):
 243.         return not self.__eq__(other)
 244. 
 245.     def get_body_encoding(self):
 246.         """Return the content-transfer-encoding used for body encoding.
 247. 
 248.         This is either the string `quoted-printable' or `base64' depending on
 249.         the encoding used, or it is a function in which case you should call
 250.         the function with a single argument, the Message object being
 251.         encoded.  The function should then set the Content-Transfer-Encoding
 252.         header itself to whatever is appropriate.
 253. 
 254.         Returns "quoted-printable" if self.body_encoding is QP.
 255.         Returns "base64" if self.body_encoding is BASE64.
 256.         Returns "7bit" otherwise.
 257.         """
 258.         assert self.body_encoding <> SHORTEST
 259.         if self.body_encoding == QP:
 260.             return 'quoted-printable'
 261.         elif self.body_encoding == BASE64:
 262.             return 'base64'
 263.         else:
 264.             return encode_7or8bit
 265. 
 266.     def convert(self, s):
 267.         """Convert a string from the input_codec to the output_codec."""
 268.         if self.input_codec <> self.output_codec:
 269.             return unicode(s, self.input_codec).encode(self.output_codec)
 270.         else:
 271.             return s
 272. 
 273.     def to_splittable(self, s):
 274.         """Convert a possibly multibyte string to a safely splittable format.
 275. 
 276.         Uses the input_codec to try and convert the string to Unicode, so it
 277.         can be safely split on character boundaries (even for multibyte
 278.         characters).
 279. 
 280.         Returns the string as-is if it isn't known how to convert it to
 281.         Unicode with the input_charset.
 282. 
 283.         Characters that could not be converted to Unicode will be replaced
 284.         with the Unicode replacement character U+FFFD.
 285.         """
 286.         if _isunicode(s) or self.input_codec is None:
 287.             return s
 288.         try:
 289.             return unicode(s, self.input_codec, 'replace')
 290.         except LookupError:
 291.             # Input codec not installed on system, so return the original
 292.             # string unchanged.
 293.             return s
 294. 
 295.     def from_splittable(self, ustr, to_output=True):
 296.         """Convert a splittable string back into an encoded string.
 297. 
 298.         Uses the proper codec to try and convert the string from Unicode back
 299.         into an encoded format.  Return the string as-is if it is not Unicode,
 300.         or if it could not be converted from Unicode.
 301. 
 302.         Characters that could not be converted from Unicode will be replaced
 303.         with an appropriate character (usually '?').
 304. 
 305.         If to_output is True (the default), uses output_codec to convert to an
 306.         encoded format.  If to_output is False, uses input_codec.
 307.         """
 308.         if to_output:
 309.             codec = self.output_codec
 310.         else:
 311.             codec = self.input_codec
 312.         if not _isunicode(ustr) or codec is None:
 313.             return ustr
 314.         try:
 315.             return ustr.encode(codec, 'replace')
 316.         except LookupError:
 317.             # Output codec not installed
 318.             return ustr
 319. 
 320.     def get_output_charset(self):
 321.         """Return the output character set.
 322. 
 323.         This is self.output_charset if that is not None, otherwise it is
 324.         self.input_charset.
 325.         """
 326.         return self.output_charset or self.input_charset
 327. 
 328.     def encoded_header_len(self, s):
 329.         """Return the length of the encoded header string."""
 330.         cset = self.get_output_charset()
 331.         # The len(s) of a 7bit encoding is len(s)
 332.         if self.header_encoding == BASE64:
 333.             return email.base64MIME.base64_len(s) + len(cset) + MISC_LEN
 334.         elif self.header_encoding == QP:
 335.             return email.quopriMIME.header_quopri_len(s) + len(cset) + MISC_LEN
 336.         elif self.header_encoding == SHORTEST:
 337.             lenb64 = email.base64MIME.base64_len(s)
 338.             lenqp = email.quopriMIME.header_quopri_len(s)
 339.             return min(lenb64, lenqp) + len(cset) + MISC_LEN
 340.         else:
 341.             return len(s)
 342. 
 343.     def header_encode(self, s, convert=False):
 344.         """Header-encode a string, optionally converting it to output_charset.
 345. 
 346.         If convert is True, the string will be converted from the input
 347.         charset to the output charset automatically.  This is not useful for
 348.         multibyte character sets, which have line length issues (multibyte
 349.         characters must be split on a character, not a byte boundary); use the
 350.         high-level Header class to deal with these issues.  convert defaults
 351.         to False.
 352. 
 353.         The type of encoding (base64 or quoted-printable) will be based on
 354.         self.header_encoding.
 355.         """
 356.         cset = self.get_output_charset()
 357.         if convert:
 358.             s = self.convert(s)
 359.         # 7bit/8bit encodings return the string unchanged (modulo conversions)
 360.         if self.header_encoding == BASE64:
 361.             return email.base64MIME.header_encode(s, cset)
 362.         elif self.header_encoding == QP:
 363.             return email.quopriMIME.header_encode(s, cset, maxlinelen=None)
 364.         elif self.header_encoding == SHORTEST:
 365.             lenb64 = email.base64MIME.base64_len(s)
 366.             lenqp = email.quopriMIME.header_quopri_len(s)
 367.             if lenb64 < lenqp:
 368.                 return email.base64MIME.header_encode(s, cset)
 369.             else:
 370.                 return email.quopriMIME.header_encode(s, cset, maxlinelen=None)
 371.         else:
 372.             return s
 373. 
 374.     def body_encode(self, s, convert=True):
 375.         """Body-encode a string and convert it to output_charset.
 376. 
 377.         If convert is True (the default), the string will be converted from
 378.         the input charset to output charset automatically.  Unlike
 379.         header_encode(), there are no issues with byte boundaries and
 380.         multibyte charsets in email bodies, so this is usually pretty safe.
 381. 
 382.         The type of encoding (base64 or quoted-printable) will be based on
 383.         self.body_encoding.
 384.         """
 385.         if convert:
 386.             s = self.convert(s)
 387.         # 7bit/8bit encodings return the string unchanged (module conversions)
 388.         if self.body_encoding is BASE64:
 389.             return email.base64MIME.body_encode(s)
 390.         elif self.body_encoding is QP:
 391.             return email.quopriMIME.body_encode(s)
 392.         else:
 393.             return s