如何在selenium中定义response要调用的回调函数并将response传给回调函数呢?
可以用中间件啊, 但是完全拖慢了scrapy的速度,现在刚开始用splash,不知道效果如何
from selenium import webdriver
from scrapy.http import HtmlResponse
class JavaScriptMiddleware(object):
def process_request(self, request, spider):
if spider.name == "some_one_use_js":
print "PhantomJS is starting..."
driver = webdriver.PhantomJS()
# driver = webdriver.Firefox()
driver.get(request.url)
time.sleep(3)
body = driver.page_source
# driver.close()
return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
else:
return