| import codecs |
| import locale |
| import re |
| import sys |
| from typing import List, Tuple |
| |
| BOMS: List[Tuple[bytes, str]] = [ |
| (codecs.BOM_UTF8, "utf-8"), |
| (codecs.BOM_UTF16, "utf-16"), |
| (codecs.BOM_UTF16_BE, "utf-16-be"), |
| (codecs.BOM_UTF16_LE, "utf-16-le"), |
| (codecs.BOM_UTF32, "utf-32"), |
| (codecs.BOM_UTF32_BE, "utf-32-be"), |
| (codecs.BOM_UTF32_LE, "utf-32-le"), |
| ] |
| |
| ENCODING_RE = re.compile(rb"coding[:=]\s*([-\w.]+)") |
| |
| |
| def auto_decode(data: bytes) -> str: |
| """Check a bytes string for a BOM to correctly detect the encoding |
| |
| Fallback to locale.getpreferredencoding(False) like open() on Python3""" |
| for bom, encoding in BOMS: |
| if data.startswith(bom): |
| return data[len(bom) :].decode(encoding) |
| # Lets check the first two lines as in PEP263 |
| for line in data.split(b"\n")[:2]: |
| if line[0:1] == b"#" and ENCODING_RE.search(line): |
| result = ENCODING_RE.search(line) |
| assert result is not None |
| encoding = result.groups()[0].decode("ascii") |
| return data.decode(encoding) |
| return data.decode( |
| locale.getpreferredencoding(False) or sys.getdefaultencoding(), |
| ) |
