url='http://www.baidu.com' while True: resp=requests.get(url).content.decode() selector=etree.HTML(resp) self.parse_data(selector) next_url=''.join(selector.xpath("//div[@class='next']/text()")) if next_url: url=next_url else: break