UTF-8のテキストをデコードする時に、
Windows-1254と判定され、文字化けする現象。
detect_enc = chardet.detect(temp)['encoding']
仕方ないので、条件分岐をひとつ増やして対応。
elif detect_enc == 'Windows-1254':
detect_enc = 'utf-8'
html_content = temp.decode(detect_enc, errors='ignore')
以下が、スクレピング部分の改良版。
import chardet
def get_html_content(url):
html_content = ''
try:
print('fetching url', url)
q = Request(url)
html = urlopen(q, timeout=15)temp = html.read()
detect_enc = chardet.detect(temp)['encoding']
if detect_enc is None:
detect_enc = 'utf-8'
elif detect_enc == 'Windows-1254':
detect_enc = 'utf-8'
html_content = temp.decode(detect_enc, errors='ignore')except Exception as e:
print('fetching url failed', url, repr(e))return html_content