7. Web Scrapping (ConsoleWriterPipeline)

import lxml.etree

import json

# receives the extract result from the spider and prints out the content

class ConsoleWriterPipeline(object):

    def open_spider(self, spider):

        None

    def close_spdier(self, spider):

        None

    def process_item(self, item, spider):

        line = json.dumps(dict(item)) + "\n"

        print(line)

        return item

import logging

import scrapy

from scrapy.crawler import CrawlerProcess

class QuotesSpider(scrapy.Spider):

    name = "quotes"

    start_urls = [

        'http://quotes.toscrape.com/page/1/',

        'http://quotes.toscrape.com/page/2/',

    ]

    custom_settings = {

        'LOG_LEVEL': logging.WARNING,  # Default : Debug

        'ITEM_PIPELINES': {'__main__.ConsoleWriterPipeline': 1}  # Used for pipeline

    }

    def parse(self, response):

        for quote in response.css('div.quote'):

            yield {

                'text': quote.css('span.text::text').get(),

                'author': quote.css('span small::text').get(),

                'tags': quote.css('div.tags a.tag::text').getall(),

            }

quotes_crawler_process = CrawlerProcess({

    'USER_AGENT': 'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1)'

})

quotes_crawler_process.crawl(QuotesSpider)

quotes_crawler_process.start()

import lxml.etree

import json

# receives the extract result from the spider and prints out the content

class ConsoleWriterPipeline(object):

    def open_spider(self, spider):

        None

    def close_spdier(self, spider):

        None

    def process_item(self, item, spider):

        line = json.dumps(dict(item)) + "\n"

        print(line)

        return item

import logging

import scrapy

from scrapy.spiders import Rule, CrawlSpider

from scrapy.linkextractors import LinkExtractor

class NewsItem(scrapy.Item):

    # define the fields for your item here like:

    headline = scrapy.Field()

    intro = scrapy.Field()

    # url = scrapy.Field()

class NewsSpider(CrawlSpider):

    name = "bbcnews"

    allowed_domains = ["bbc.co.uk"]

    start_urls = ["http://www.bbc.co.uk/news/technology/", ]

    custom_settings = {

        'LOG_LEVEL': logging.WARNING,

        'ITEM_PIPELINES': {'__main__.ConsoleWriterPipeline': 1}  # Used for pipeline

    }

    rules = [Rule(LinkExtractor(allow=['/technology-\d+']), 'parse_story')]

    def parse_story(self, response):

        story = NewsItem()

        story['headline'] = response.xpath('.//h1[@class="story-body__h1"]/text()').get()

        story['intro'] = response.xpath('.//p[@class="story-body__introduction"]/text()').get()

        yield {

            "headline": story['headline'],

            "intro": story['intro']

        }

from scrapy.crawler import CrawlerProcess

hgw_crawler_process = CrawlerProcess({

    'USER_AGENT': 'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1)'

})

hgw_crawler_process.crawl(NewsSpider)

hgw_crawler_process.start()

import lxml.etree

import json

# receives the extract result from the spider and prints out the content

class ConsoleWriterPipeline(object):

    def open_spider(self, spider):

        None

    def close_spdier(self, spider):

        None

    def process_item(self, item, spider):

        line = json.dumps(dict(item)) + "\n"

        print(line)

        return item

import logging

import scrapy

from scrapy.spiders import CrawlSpider, Rule

from scrapy.linkextractors import LinkExtractor

class BooksCrawlSpider(CrawlSpider):

    name = 'books-crawlspider'

    allowed_domains = ['toscrape.com']

    start_urls = ['http://books.toscrape.com']

    custom_settings = {

      'LOG_LEVEL': logging.WARNING,

      'ITEM_PIPELINES': {'__main__.ConsoleWriterPipeline': 1}#, # Used for pipeline

      }

    rules = [

        Rule(

            LinkExtractor(allow=('/catalogue/page-\d+.html')),follow=True

        ),

        Rule(

            # LinkExtractor(deny=('/category/books', '/catalogue/page-\d+.html', '.com/index.html')),

            LinkExtractor(deny=('/category/books', '.com/index.html')),callback='parse_book_page',

            follow=True

        ),

    ]

    def parse_book_page(self, response):

      yield {

          'title': response.css('.product_main h1::text').get(),

          'price': response.css('.product_main p.price_color::text').re_first('£(.*)'),

          'stock': int(''.join(response.css('.product_main p.instock.availability::text').re('(\d+)')))

          }

from scrapy.crawler import CrawlerProcess

hgw_crawler_process = CrawlerProcess({

    'USER_AGENT': 'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1)'

})

hgw_crawler_process.crawl(BooksCrawlSpider)

hgw_crawler_process.start()