From 303c8a7584be47dedbee5a2270be1134dcbffbfa Mon Sep 17 00:00:00 2001 From: Jonathan Harker Date: Mon, 23 Nov 2015 11:52:45 +1300 Subject: [PATCH] Update to work with Python 3 - Replace Mechanize with BeautifulSoup. - Adjust requirements.txt for updated dependencies. --- models.py | 15 +++++++++++---- requirements.txt | 11 +++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/models.py b/models.py index 9717a8f..fe0511c 100644 --- a/models.py +++ b/models.py @@ -1,5 +1,13 @@ -from mechanize import Browser +#! /usr/bin/env python +""" +Database models for lolbot. +""" + +from __future__ import print_function, unicode_literals + +import requests from datetime import datetime +from bs4 import BeautifulSoup from sqlalchemy import (Column, String, Text, Integer, DateTime) from sqlalchemy.ext.declarative import (declarative_base) @@ -59,9 +67,8 @@ class Url(Model): # populate the title from the URL if not given. if title is None: try: - br = Browser() - br.open(self.url) - self.title = br.title() + dom = BeautifulSoup(requests.get(self.url), 'html.parser') + self.title = dom.title.string except Exception: self.title = '' diff --git a/requirements.txt b/requirements.txt index 1401a04..3eb1e70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,4 @@ SQLAlchemy==1.0.9 -argparse==1.2.1 -irc==11.0.1 -jaraco.timing==1.1.5 -jaraco.util==10.6 -mechanize==0.2.5 -more-itertools==2.2 -six==1.10.0 -wsgiref==0.1.2 +beautifulsoup4==4.4.1 +irc==13.2 +requests==2.8.1