python - Rastreo de datos con éxito pero no puede raspar o escribir en csv

web-scraping scrapy

Question

He agregado un DOWNLOAD_DELAY = 2 y un COOKIES_ENABLED = False , mi spider rastrea el sitio web pero no escribe los elementos en mi archivo CSV. No creo que sea normal porque cuando no agrego estas dos configuraciones, todo está bien ... ¿Podría alguien ayudarme por favor?

Llamo a mi araña con esta línea en mi símbolo del sistema: scrapy crawl CDiscount -o items.csv

Aquí está mi araña:

# -*- coding: utf-8 -*-
# Every import is done for a specific use
import scrapy                                       # Once you downloaded scrapy, you have to import it in your code to use it.
import re                                           # To use the .re() function, which extracts just a part of the text you crawl. It''s using regex (regular expressions)
import numbers                                      # To use mathematics things, in this case : numbers.
from cdiscount_test.items import CdiscountTestItem  # To return the items you want. Each item has a space allocated in the momery, created in the items.py file, which is in the cdiscount_test directory.
from urllib.parse import urljoin                    # To use the urljoin function, which make that an url is readable by the spider.
from scrapy.spiders import CrawlSpider, Rule        # To use rules and LinkExtractor, which allowed the spider to follow every url on the page you crawl.
from scrapy.linkextractors import LinkExtractor     # Look above.

# Your spider
class CdiscountsellersspiderSpider(CrawlSpider):
    name = ''CDiscount''             # The name of your spider. You call it in the anaconda prompt.
    handle_httpstatus_list = [502, 503]
    allowed_domains = [''cdiscount.com'']         # Web domains allowed by you, your spider cannot enter on a page which is not in that domain.
    start_urls = [''http://www.cdiscount.com'']   # The first link you crawl.

    # To allow your spider to follow the urls that are on the actual page.
    rules = (
        Rule(LinkExtractor(allow=()), callback=''parse_1''),
    )

    # Your "crawl" function
    def parse_1(self, response):
        item = CdiscountTestItem() # The spider now knowws that the items you want have to be stored in the item variable.

        # Extraction of the data you are looking for by following their paths in the HTML code.
        # response.xpath() is to find the location of the data you want (the HTML code) and .extract() or .re() is to extract the data from the HTML code.
        name = response.xpath(''//div[@class="shtName"]/div[@class="shtOver"]/h1[@itemprop="name"]/text()'').extract()
        country = response.xpath(''//div[@class="shtName"]/span[@class="shTopCExp"]/text()'').extract()

        # If your data are on the page, you pull them out
        # To know if they are, you check their lenght, which means : if the variable in which you put your data are empty, then the data are obviously not on that page.
        if ((len(name) != 0) & (len(country) != 0)):
            sales = response.xpath(''//div[@class="shcSell"]//span//text()'').re(r''([/d]*).*'')
            nbproducts_tot = response.xpath(''//div[@class="jsFacetListing mgFacetListing mgFOpen"][1]/div[@class="mgFAllList"][1]/ul/li/label/span/text()'').re(r''/(([/d]*)/)'')
            avgcost_tot = response.xpath(''//div[@class="jsFacetListing mgFacetListing mgFOpen"][1]/div[@class="mgFAllList"][1]/ul/li/label/span/text()'').re(r''/<?(.*[/d]*)/€+'')
            id_tmp = response.xpath(''//ul[@class="sCgvLegal"]/li/span[contains(.,"Immatriculation")]/following-sibling::text()'').extract()
            address = response.xpath(''//ul[@class="sCgvLegal"]/li/span[contains(.,"Adresse")]/following-sibling::text()'').extract()
            business = response.xpath(''//ul[@class="sCgvLegal"]/li/span[contains(.,"Sociale")]/following-sibling::text()'').extract()

            # You calculate the avergae cost, the number of all the products the seller has and the turnover of the lasts 12 months.
            # First, the average cost :
                # avgcost is a list of list. Each "under list" contains all the slices of price which exist for the products of this seller.
                # You order the slice of price and the number of products they contain.
                # Then, you sum the max and min of each slice of price and divide this number by two (or let it like that, depending if the slice is between two numbers or just "under 10" or "above 1000").
                    # Like that, you have the average cost of each slice of price.
            avgcost = [avgcost_tot[i].split('' '') for i in range(len(avgcost_tot))]
            if (len(avgcost) != 0):
                for i in range(len(avgcost)):
                    avgcost[i].remove(avgcost[i][len(avgcost[i])-1])
                if len(avgcost[0]) != 3:
                    if len(avgcost[len(avgcost)-1]) != 3:
                        for i in range(1,len(avgcost)-1):
                            avgcost[i].remove(avgcost[i][1])
                            avgcost[i] = sum(map(int, avgcost[i]))/len(avgcost[i])
                        avgcost[len(avgcost)-1] = sum(map(int, avgcost[len(avgcost)-1]))/len(avgcost[len(avgcost)-1])
                    else:
                        for i in range(1, len(avgcost)):
                            avgcost[i].remove(avgcost[i][1])
                            avgcost[i] = sum(map(int, avgcost[i]))/len(avgcost[i])
                    avgcost[0] = sum(map(int, avgcost[0]))/len(avgcost[0])
                else:
                    if len(avgcost[len(avgcost)-1]) != 3:
                        for i in range(0,len(avgcost)-1):
                            avgcost[i].remove(avgcost[i][1])
                            avgcost[i] = sum(map(int, avgcost[i]))/len(avgcost[i])
                        avgcost[len(avgcost)-1] = sum(map(int, avgcost[len(avgcost)-1]))/len(avgcost[len(avgcost)-1])
                    else:
                        for i in range(0, len(avgcost)):
                            avgcost[i].remove(avgcost[i][1])
                            avgcost[i] = sum(map(int, avgcost[i]))/len(avgcost[i])

                # You pull out the number of products of each slice of price and put these info in a list.
                nbproducts_list = list(map(int, nbproducts_tot))
                # You divide each average cost of each slice of price by the number of products contains in that slice.
                    # Now, you have a list of every average cost by product, by slice of price.
                cost_list = [avgcost[i]*nbproducts_list[i] for i  in range(len(avgcost))]

                # The total number of products sold by the seller.
                nbproducts = sum(nbproducts_list)
                # The average cost of one product sold by this seller.
                cost = int(sum(cost_list)/nbproducts)

                item[''Average_cost''] = ''''.join(str(cost)).strip()
                item[''Products_number''] = ''''.join(str(nbproducts)).strip()

                if (len(sales) != 0):
                    # Turnover of the lasts 12 months (average cost of one product by the number of products sold on the lasts 12 months).
                    turnover = cost*int(sales[0])

                    item[''TurnOver_12months''] = ''''.join(str(turnover)).strip()

            # Store the data at the right place in your output file
            item[''Storefront_name''] = ''''.join(name).strip()
            item[''Business_name''] = ''''.join(business).strip()
            item[''Address''] = ''''.join(address).strip()
            item[''Country''] = ''''.join(country).strip()
            item[''ID''] = ''''.join(id_tmp).strip()
            item[''Sales_12months''] = ''''.join(sales).strip()
            item[''url_seller''] = ''''.join(str(response.url))
            # Return the data
            yield item

        # If not, there was no data on the page and you have to find all the links on your page and launch the first function on them.
        else:
            for sel in response.xpath(''//html/body''):
                list_urls = sel.xpath(''//a/@href'').extract()
                list_iframe = response.xpath(''//div[@class="ProductPriceBox-item detail"]/div/a/@href'').extract()
                if (len(list_iframe) != 0):
                    for list_iframe in list_urls:
                        yield scrapy.Request(list_iframe, callback=self.parse_start_url, meta={''dont_redirect'': True})
                for url in list_urls:
                    yield scrapy.Request(response.urljoin(url), callback=self.parse_start_url, meta={''dont_redirect'': True})

Y aquí está mi configuración:

BOT_NAME = ''cdiscount_test''

SPIDER_MODULES = [''cdiscount_test.spiders'']
NEWSPIDER_MODULE = ''cdiscount_test.spiders''

DOWNLOAD_DELAY = 2
COOKIES_ENABLED = False

EDITAR:

Ahora, mi araña ni siquiera destruye el sitio web. Simplemente rastrea todo, busca todos los enlaces a seguir, pero no recuperas los datos y no entiendo por qué. No cambié nada desde la última vez que funcionó.

¡¡Gracias!!