Scrapy回調函數不抓取整個數據？這一切

Question

首先是我的代碼 - ：

from twisted.internet import reactor 
from scrapy.crawler import CrawlerProcess, CrawlerRunner 
import scrapy 
#from scrapy import log, signals 
from scrapy.utils.log import configure_logging 
from scrapy.utils.project import get_project_settings 
from scrapy.settings import Settings 
import datetime 
from multiprocessing import Process, Queue 
import os 
from scrapy.http import Request 
from scrapy import signals 
from scrapy.xlib.pydispatch import dispatcher 
from scrapy.signalmanager import SignalManager 
import re 

#query=raw_input("Enter a product to search for= ") 
query='apple' 
query1=query.replace(" ", "+") 


class DmozItem(scrapy.Item): 

    productname = scrapy.Field() 
    product_link = scrapy.Field() 
    current_price = scrapy.Field() 
    mrp = scrapy.Field() 
    offer = scrapy.Field() 
    imageurl = scrapy.Field() 
    outofstock_status = scrapy.Field() 
    add = scrapy.Field() 

class DmozSpider(scrapy.Spider): 
    name = "dmoz" 
    allowed_domains = ["http://www.bestmercato.com"] 


    def start_requests(self): 

     task_urls = [ 
     ] 
     i=1 
     for i in range(1,2): 
      temp=("https://www.bestmercato.com/index.php?route=product/search&search="+query1+"&page="+str(i)) 
      task_urls.append(temp) 
      i=i+1 

     start_urls = (task_urls) 
#  p=len(task_urls) 
     return [ Request(url = start_url) for start_url in start_urls ] 


    def parse(self, response): 
     items = [] 

     for sel in response.xpath('//html/body/div/div/div[4]/div/div/div[5]/div'): 

      item = DmozItem() 

      item['productname'] = str(sel.xpath('div[@class="product-thumb"]/div[@class="small_detail"]/div[@class="name"]/a/text()').extract())[3:-2] 

      item['product_link'] = str(sel.xpath('div[@class="product-thumb"]/div[@class="small_detail"]/div[@class="name"]/a/@href').extract())[3:-2] 

      point1 = sel.xpath('div[@class="product-thumb"]/div[@class="small_detail"]/div[4]').extract() 
      point = str(sel.xpath('div[@class="product-thumb"]/div[@class="small_detail"]/div[4]/@class').extract())[3:-2] 
      checker = "options" in point 
      item['current_price'] = "" 
      if checker: 
       i=1 
       p=1 
       while i==1: 
        t = str(sel.xpath('div[@class="product-thumb"]/div[@class="small_detail"]/div[4]/div/select/option['+str(p)+']/text()').extract())[3:-2] 
        #print t   
        if 'Rs' not in t: 
         i = 2 
        elif 'Rs' in t: 
         i = 1 
        t= " ".join(t) 
        s = t.translate(None, '\ t')[:-2] 
        item['current_price'] = item['current_price'] + ' ; ' + s 
        p = p+1 
       item['current_price'] = item['current_price'][3:-3] 

      else: 
       item['current_price'] = 'Rs. ' + str(sel.xpath('div[@class="product-thumb"]/div[@class="small_detail"]/div[not (@class="name") or not(@class="description") or not(@class="qty") or not(@class="box_btn_icon")]/text()').extract())[46:-169] 
       re.findall(r"[-+]?\d*\.\d+|\d+", item["current_price"]) 

      try: 
       test1 = str(sel.xpath('div/div[2]/div[3]/span[1]/text()').extract())[3:-2] 
       _digits = re.compile('\d') 
       if bool(_digits.search(test1)): 
        print 'hi' 
        test1=test1[:2]+'. '+test1[3:] 
        item['mrp'] = test1 
        #item['mrp'][2:2]='.' 
        test2 = str(sel.xpath('div/div[2]/div[3]/span[2]/text()').extract())[3:-2] 
        test2=test2[:2]+'. '+test2[3:] 
        item['current_price']=test2 

       else: 
        item['mrp'] = item['current_price']     
      except: 
       item['mrp'] = item['current_price'] 

      item['offer'] = 'No additional offer available' 

      item['imageurl'] = str(sel.xpath('div[@class="product-thumb"]/div[@class="image"]/a[not (@class="sft_quickshop_icon")]/img[@class="img-responsive"]/@src').extract())[3:-2] 

      item['outofstock_status'] = str('In Stock') 

      request = Request(str(item['product_link']),callback=self.parse2, dont_filter=True) 
      request.meta['item'] = item 
#   print item 
      items.append(item) 
      return request 

     print (items) 

    def parse2(self, response): 

     item = response.meta['item'] 
     item['add'] = response.url 
     return item 

spider1 = DmozSpider() 
settings = Settings() 
settings.set("PROJECT", "dmoz") 
settings.set("CONCURRENT_REQUESTS" , 100) 
#) 
#settings.set("DEPTH_PRIORITY" , 1) 
#settings.set("SCHEDULER_DISK_QUEUE" , "scrapy.squeues.PickleFifoDiskQueue") 
#settings.set("SCHEDULER_MEMORY_QUEUE" , "scrapy.squeues.FifoMemoryQueue") 
crawler = CrawlerProcess(settings) 
crawler.crawl(spider1) 
crawler.start() 

現在，這些都是我所面臨的問題。

1.有很多div可以通過這個xpath找到 - '// html/body/div/div/div [4]/div/div/div [5]/div'。但是，上面的代碼只會刪除第一個div的內容，即具有xpath'html/body/div/div/div [4]/div/div/div [5]/div [1]'，而不是他們全部。

的那一刻我評論這三條線，刮板刮下的一切，那麼顯然我不能添加在本期特價貨品「添加」字段：

request = Request(str(item['product_link']),callback=self.parse2, dont_filter=True) 
request.meta['item'] = item 
return request 

所以，我想湊所有除了我的項目類中的'添加'字段（注意類DmozItem）。我怎麼做？請爲我的特定情況提供更正的代碼，這將是最好的方式！其次，正如我所說的，當我評論上面提到的三條線時，程序會在接近5秒（大約4.9秒）的時間內擦除所有內容。但是，只要我取消註釋，那3行（再次提到上面提到的那些行），程序的運行時間就會大大超過，並且運行時間接近9秒（大約8.8 - 8.9秒） 。爲什麼會發生？那是因爲這個 - dont_filter=True？請提出解決方法，因爲運行時間對我來說可能是一個非常大的問題。另外，我可以以某種方式減少5秒（約4.9）的初始時間嗎？

Answer 1

使用html/body/div/div/div[4]/div/div/div[5]//div得到所有div秒後div[5]。

編輯： 這是正確的XPath - //html/body/div/div/div[4]/div/div/div[5]/div，這給了所有的div的DIV之後[5]。前面提到的一個，給了多個錯誤！

如果在循環中執行return語句，則會結束整個方法的執行。因此，如果啓用這三行，則會在第一個元素後結束執行方法（和for循環）。

這意味着您應該yield您的請求，而不是return它。