Step by Step of "Web scraping with Python" ----Richard Lawson ---2/n

1. Debian 9 + Python 3.5.3

"python3 --version"

link_crawler3.py

# -*- coding: utf-8 -*-

import re
import queue
import time
from common import download
from urllib import request 
from urllib import robotparser
from urllib.parse import urljoin
from urllib.parse import urlparse
from datetime import datetime


def link_crawler(seed_url, link_regex=None, delay=5, max_depth=-1, max_urls=-1, headers=None, user_agent='wswp', proxy=None, num_retries=1):
    """
    Crawl from the given seed URL following links matched by link_regex
    """
    # the queue of URL's that still need to be crawled
    crawl_queue = queue.deque([seed_url])
    # the URL's that have been seen and at what depth
    seen = {seed_url: 0}
    # track how many URL's have been downloaded
    num_urls = 0
    rp = Throttle.get_robots(seed_url)
    throttle = Throttle(delay)
    headers = headers or {}
    if user_agent:
        headers['User-agent'] = user_agent


    while crawl_queue:
        url = crawl_queue.pop()
        # check url passes robots.txt restrictions
        if rp.can_fetch(user_agent, url):
            throttle.wait(url)
            html = download(url, headers, proxy=proxy, num_retries=num_retries)
            links = []

            depth = seen[url]
            if depth != max_depth:
                # can still crawl further
                if link_regex:
                    # filter for links matching our regular expression
                    links.extend(link for link in get_links(html) if re.match(link_regex, link))


                    for link in links:
                        link = normalize(seed_url, link)
                        # check whether already crawled this link
                        if link not in seen:
                            seen[link] = depth + 1
                            # check link is within same domain
                            if same_domain(seed_url, link):
                                # success! add this new link to queue
                                crawl_queue.append(link)


            # check whether have reached downloaded maximum
            num_urls += 1
            if num_urls == max_urls:
                break
            else:
                print('Blocked by robots.txt:%s'%url)



class Throttle:
    """
    Throttle downloading by sleeping between requests to same domain
    """
    def __init__(self, delay):
        # amount of delay between downloads for each domain
        self.delay = delay
        # timestamp of when a domain was last accessed
        self.domains = {}


    def wait(self, url):
        domain = urlparse(url).netloc
        last_accessed = self.domains.get(domain)


        if self.delay > 0 and last_accessed is not None:
            sleep_secs = self.delay - (datetime.now() - last_accessed).seconds
            if sleep_secs > 0:
                time.sleep(sleep_secs)
        self.domains[domain] = datetime.now()


    def download(url, headers, proxy, num_retries, data=None): 
        print('Downloading:%s'%url)
        request = request.Request(url, data, headers)
        opener = request.build_opener()
        if proxy:
            proxy_params = {urlparse.urlparse(url).scheme: proxy}
            opener.add_handler(request.ProxyHandler(proxy_params))

        try:
            response = opener.open(request)
            html = response.read()
            code = response.code

        except request.URLError as e:
            print('Download error:%s'%e.reason)
            html = ''
            if hasattr(e, 'code'):
                code = e.code
                if num_retries > 0 and 500 <= code < 600:
                    # retry 5XX HTTP errors
                    return download(url, headers, proxy, num_retries-1, data)
                else:
                    code =None

            return html


    def normalize(seed_url, link):
        """
        Normalize this URL by removing hash and adding domain
        """
        link, _ = urlparse.urldefrag(link) # remove hash to avoid duplicates
        return urlparse.urljoin(seed_url, link)

    def same_domain(url1, url2):
        """
        Return True if both URL's belong to same domain
        """
        return urlparse.urlparse(url1).netloc == urlparse.urlparse(url2).netloc

    def get_robots(url):
        """
        Initialize robots parser for this domain
        """
        rp = robotparser.RobotFileParser()
        rp.set_url(urljoin(url, '/robots.txt'))
        rp.read()
        return rp

    def get_links(html):
        """
        Return a list of links from html 
        """
        # a regular expression to extract all links from the webpage
        webpage_regex = re.compile('<a[^>]+href=["\'](.*?)["\']', re.IGNORECASE)
        # list of all links from the webpage
        return webpage_regex.findall(html)

if __name__ == '__main__':
    link_crawler('http://example.webscraping.com', '/(index|view)', delay=0, num_retries=1, user_agent='BadCrawler')
    link_crawler('http://example.webscraping.com', '/(index|view)', delay=0, num_retries=1, max_depth=1, user_agent='GoodCrawler')

　　

　　2.  it seems we should learn more about "urllib"

cor@debian:~$ /usr/bin/python3 /home/cor/webscrappython/Web_Scraping_with_Python/chapter01/link_crawler3.py
Traceback (most recent call last):
  File "/home/cor/webscrappython/Web_Scraping_with_Python/chapter01/link_crawler3.py", line 147, in <module>
    link_crawler('http://example.webscraping.com', '/(index|view)', delay=0, num_retries=1, user_agent='BadCrawler')
  File "/home/cor/webscrappython/Web_Scraping_with_Python/chapter01/link_crawler3.py", line 22, in link_crawler
    rp = Throttle.get_robots(seed_url)
  File "/home/cor/webscrappython/Web_Scraping_with_Python/chapter01/link_crawler3.py", line 133, in get_robots
    rp.set_url(urlparse.urljoin(url, '/robots.txt'))
AttributeError: 'function' object has no attribute 'urljoin'

　　

3.  

TypeError: expected string or bytes-like object

cor@debian:~$ /usr/bin/python3 /home/cor/webscrappython/Web_Scraping_with_Python/chapter01/link_crawler3.py
Downloading:http://example.webscraping.com
Downloading--2
Downloading:http://example.webscraping.com
Downloading --- 5
Downloading:http://example.webscraping.com
Downloading --- 5
Traceback (most recent call last):
  File "/home/cor/webscrappython/Web_Scraping_with_Python/chapter01/link_crawler3.py", line 150, in <module>
    link_crawler('http://example.webscraping.com', '/(index|view)', delay=0, num_retries=1, max_depth=1, user_agent='GoodCrawler')
  File "/home/cor/webscrappython/Web_Scraping_with_Python/chapter01/link_crawler3.py", line 36, in link_crawler
    html = download(url, headers, proxy=proxy, num_retries=num_retries)
  File "/home/cor/webscrappython/Web_Scraping_with_Python/chapter01/common.py", line 72, in download5
    html = opener.open(requestnew).read().decode('utf-8')
  File "/usr/lib/python3.5/urllib/request.py", line 466, in open
    response = self._open(req, data)
  File "/usr/lib/python3.5/urllib/request.py", line 484, in _open
    '_open', req)
  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
    result = func(*args)
  File "/usr/lib/python3.5/urllib/request.py", line 1282, in http_open
    return self.do_open(http.client.HTTPConnection, req)
  File "/usr/lib/python3.5/urllib/request.py", line 1254, in do_open
    h.request(req.get_method(), req.selector, req.data, headers)
  File "/usr/lib/python3.5/http/client.py", line 1107, in request
    self._send_request(method, url, body, headers)
  File "/usr/lib/python3.5/http/client.py", line 1147, in _send_request
    self.putheader(hdr, value)
  File "/usr/lib/python3.5/http/client.py", line 1083, in putheader
    if _is_illegal_header_value(values[i]):
TypeError: expected string or bytes-like object

　　4.   below is "cpmmon"

 

# -*- coding: utf-8 -*-

from urllib import  request


def download1(url):
    """Simple downloader"""
    # before
    #return urllib.urlopen(url).read()
    #after, using urllib.request instead
    print('Downloading--1')
    return request.urlopen(url)

from urllib import  request
def download2(url):
    """Download function that catches errors"""
    print('Downloading:%s'%url)
    print('Downloading--2')
    try:
        html = request.urlopen(url).read()
    except request.URLError as e:
        print('Download error:%s'%e.reason)
        html = None
    return html
download2('http://example.webscraping.com')

def download3(url, num_retries=2):
    """Download function that also retries 5XX errors"""
    print('Downloading:%s'%url)
    print('Downloading--3')
    try:
        html = request.urlopen(url).read()
    except request.URLError as e:
        print('Download error:%s'% e.reason)
        html = None
        if num_retries > 0:
            if hasattr(e, 'code') and 500 <= e.code < 600:
                # retry 5XX HTTP errors
                html = download3(url, num_retries-1)
    return html


def download4(url, user_agent='wswp', num_retries=2):
    """Download function that includes user agent support"""
    print('Downloading:%s'%url)
    print('Downloading--4')
    headers = {'User-agent': user_agent}
    requestnew = request.Request(url, headers=headers)
    try:
        html = request.urlopen(requestnew).read()
    except request.URLError as e:
        print('Download error:%s'%e.reason)
        html = None
        if num_retries > 0:
            if hasattr(e, 'code') and 500 <= e.code < 600:
                # retry 5XX HTTP errors
                html = download4(url, user_agent, num_retries-1)
    return html


def download5(url, user_agent='wswp', proxy=None, num_retries=2):
    """Download function with support for proxies"""
    print('Downloading:%s'%url)
    print('Downloading --- 5')
    headers = {'User-agent': user_agent}
    requestnew = request.Request(url, headers=headers)
    opener = request.build_opener()
    if proxy:
        proxy_params = {urlparse.urlparse(url).scheme: proxy}
        opener.add_handler(request.ProxyHandler(proxy_params))
    try:
        html = opener.open(requestnew).read().decode('utf-8')
    except request.URLError as e:
        print('Download error:%s'%e.reason)
        html = None
        if num_retries > 0:
            if hasattr(e, 'code') and 500 <= e.code < 600:
                # retry 5XX HTTP errors
                html = download5(url, user_agent, proxy, num_retries-1)
    return html


download = download5


if __name__ == '__main__':
    print(download('http://example.webscraping.com'))

5.　　

 

>>> import urllib.parse
>>> import urllib.request
>>> url = 'http://www.someserver.com/cgi-bin/register.cgi'
>>> user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'\
... 
KeyboardInterrupt
>>> user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'
>>> url
'http://www.someserver.com/cgi-bin/register.cgi'
>>> values = {'name': 'Michael Foord',
...           'location': 'Northampton',
...           'language': 'Python' }
>>> headers = {'User-Agent': user_agent}

>>> data=urllib.parse.urlencode(values)
>>> data
'location=Northampton&language=Python&name=Michael+Foord'
>>> data = data.encode('ascii')
>>> data
b'location=Northampton&language=Python&name=Michael+Foord'
>>> req = urllib.request.Request(url, data, headers)
>>> with urllib.request.urlopen(req) as response:
... the_page = response.read()
... print(the_page)

 

b'<!DOCTYPE html><html><head><meta http-equiv="x-ua-compatible" content="IE=edge"><title></title><script type="text/javascript">(function() {var p = "eyJ1cmkiOiIvY2dpLWJpbi9yZWdpc3Rlci5jZ2kiLCJhcmdzIjoiIiwicmVmZXJlciI6IiJ9:1jHNCg:E4Xczfh7oF8UHMBAouFg0z9KGN8", as = "http://www.someserver.com/mtm/async/", f = "http://www.someserver.com/mtm/direct/";function d(n){window.location.href = "http://www42.someserver.com/"+n;}function ar(r) {if (r.slice(0, 1) !== ".") {try {window.location.assign(r);} catch (err) {}try {var mar = document.createElement("meta");mar.httpEquiv = "refresh";mar.content = "0;url="+r;document.getElementsByTagName("head")[0].appendChild(mar);} catch (err) {}} else {var s = document.createElement("span");s.id="ecode";s.appendChild(document.createTextNode(r.slice(1)));document.getElementsByTagName("body")[0].appendChild(s);}}if ("fetch" in window) {try {fetch(as + p + "/1", {credentials: "include"}).then(function(r) {if (!r.ok) {throw Error("50x");}return r.text();}).then(function(r) {ar(r);});} catch (err) {d(2);}} else {try {var x = new XMLHttpRequest();x.open("GET", as + p + "/2", false);x.onerror = function() {d(3);};x.onload = function() {if (x.status === 200) {ar(x.responseText);} else {d(4);}};x.onreadystatechange = function(r) {if (x.readyState === 4){if (x.status === 200) {ar(x.responseText);} else {d(6);}}};x.send();} catch (err) {d(5);}}})();</script><meta http-equiv="refresh" content="5;url=http://www.someserver.com/mtm/direct/eyJ1cmkiOiIvY2dpLWJpbi9yZWdpc3Rlci5jZ2kiLCJhcmdzIjoiIiwicmVmZXJlciI6IiJ9:1jHNCg:E4Xczfh7oF8UHMBAouFg0z9KGN8/1" /></head><body></body></html>'

>>>