8af477fca8
Changes to be committed: - kleinere Korrekturen/Verbesserungen modified: homepage/redesign2018/markdownExperiment/Makefile modified: homepage/redesign2018/markdownExperiment/src/jsonSd/cwsvJudo.json - Neuer Ansatz für den Wettkampfsammler new file: wkOrg/src/wkScraper/scrapyDocAuthorSpider.py new file: wkOrg/src/wkScraper/scrapyDocQuoteSpider.py new file: wkOrg/src/wkScraper/scrapyJvsKalender.py
27 lines
823 B
Python
27 lines
823 B
Python
import scrapy
|
|
|
|
|
|
class AuthorSpider(scrapy.Spider):
|
|
name = 'author'
|
|
|
|
start_urls = ['http://quotes.toscrape.com/']
|
|
|
|
def parse(self, response):
|
|
# follow links to author pages
|
|
for href in response.css('.author + a::attr(href)'):
|
|
yield response.follow(href, self.parse_author)
|
|
|
|
# follow pagination links
|
|
for href in response.css('li.next a::attr(href)'):
|
|
yield response.follow(href, self.parse)
|
|
|
|
def parse_author(self, response):
|
|
def extract_with_css(query):
|
|
return response.css(query).extract_first().strip()
|
|
|
|
yield {
|
|
'name': extract_with_css('h3.author-title::text'),
|
|
'birthdate': extract_with_css('.author-born-date::text'),
|
|
'bio': extract_with_css('.author-description::text'),
|
|
}
|