def parse_item(self, response):</p><br /> <p> loader = ItemLoader(GaokaopaiZhuanyeItem(), response)</p><br /> <p> loader.add_value(‘url’, response.url)<br /><br /> loader.add_css(‘name’, u’.majorTitle>h1::text’)</p><br /> <p> loader.add_xpath(‘code’, u’//div[@class=”majorBase”]/h3[starts-with(., “?????”)]/text()’, re=ur’?(.+)’)<br /><br /> loader.add_xpath(‘degree’, u’//div[@class=”majorBase”]/h3[starts-with(., “?????”)]/text()’, re=ur’?(.+)’)<br /><br /> loader.add_xpath(‘period’, u’//div[@class=”majorBase”]/h3[starts-with(., “?????”)]/text()’, re=ur’?(.+)’)<br /><br /> loader.add_xpath(‘courses’, u’//div[@class=”course”]/h3[.=”?????”]/following-sibling::p/text()’)</p><br /> <p> def parse_related():</p><br /> <p> for e in response.xpath(u’//div[@class=”course”]/h3[.=”?????”]/following-sibling::a’):<br /><br /> yield {<br /><br /> ‘url’: e.css(‘::attr(href)’).extract_first(),<br /><br /> ‘code’: e.css(‘::attr(href)’).re_first(ur’-([^-]+)\.html’),<br /><br /> ‘name’: e.css(‘::text’).extract_first(),<br /><br /> }</p><br /> <p> loader.add_value(‘related’, list(parse_related()))</p><br /> <p> def parse_category():</p><br /> <p> category = []</p><br /> <p> for i in [u”????”, u”????”, u”????”]:<br /><br /> x = u’//h3[.=”{}”]/following-sibling::ul[1]/li[@class=”current”]/a’.format(i)<br /><br /> e = response.xpath(x)<br /><br /> category.append({<br /><br /> ‘url’: e.css(‘::attr(href)’).extract_first(),<br /><br /> ‘code’: e.css(‘::attr(href)’).re_first(ur’/zhuanye([-0-9]*)\.html’).strip(‘-‘),<br /><br /> ‘name’: e.css(‘::text’).extract_first(),<br /><br /> })</p><br /> <p> return category</p><br /> <p> loader.add_value(‘category’, parse_category())<br /><br /> loader.add_css(‘detail’, u’.majorCon’)</p><br /> <p> item = loader.load_item()</p><br /> <p> return Request(<br /><br /> url=”http://www.gaokaopai.com/zhuanye-jiuye-{}.html”.format(item[‘code’][0]),<br /><br /> meta={‘item’: item},<br /><br /> callback=self.parse_jiuye<br /><br /> )

def parse_question(self, response):<br /><br /> # ??question??? ??????????question item<br /><br /> if “QuestionHeader-title” in response.text:<br /><br /> # ?????<br /><br /> match_obj = re.match(“(.*zhihu.com/question/(\d+))(/|$).*”, response.url)<br /><br /> if match_obj:<br /><br /> question_id = int(match_obj.group(2))</p><br /> <p> item_loader = ItemLoader(item=ZhihuQuestionItem(), response=response)<br /><br /> item_loader.add_css(“title”, “h1.QuestionHeader-title::text”)<br /><br /> item_loader.add_css(“content”, “.QuestionHeader-detail”)<br /><br /> item_loader.add_value(“url”, response.url)<br /><br /> item_loader.add_value(“zhihu_id”, question_id)<br /><br /> item_loader.add_css(“answer_num”, “.List-headerText span::text”)<br /><br /> item_loader.add_css(“comments_num”, “.QuestionHeader-Comment button::text”)<br /><br /> item_loader.add_css(“watch_user_num”, “.NumberBoard-value::text”)<br /><br /> item_loader.add_css(“topics”, “.QuestionHeader-topics .Popover div::text”)</p><br /> <p> question_item = item_loader.load_item()<br /><br /> else:<br /><br /> # ????????item??<br /><br /> match_obj = re.match(“(.*zhihu.com/question/(\d+))(/|$).*”, response.url)</p><br /> <p> if match_obj:<br /><br /> question_id = int(match_obj.group(2))</p><br /> <p> item_loader = ItemLoader(item=ZhihuQuestionItem(), response=response)<br /><br /> # item_loader.add_css(“title”, “.zh-question-title h2 a::text”)<br /><br /> item_loader.add_xpath(“title”,<br /><br /> “//*[@id=’zh-question-title’]/h2/a/text()|//*[@id=’zh-question-title’]/h2/span/text()”)<br /><br /> item_loader.add_css(“content”, “#zh-question-detail”)<br /><br /> item_loader.add_value(“url”, response.url)<br /><br /> item_loader.add_value(“zhihu_id”, question_id)<br /><br /> item_loader.add_css(“answer_num”, “#zh-question-answer-num::text”)<br /><br /> item_loader.add_css(“comments_num”, “#zh-question-meta-wrap a[name=”addcomment”]::text”)<br /><br /> # item_loader.add_css(“watch_user_num”, “#zh-question-side-header-wrap::text”)<br /><br /> item_loader.add_xpath(“watch_user_num”,<br /><br /> “//*[@id=’zh-question-side-header-wrap’]/text()|//*[@class=”zh-question-followers-sidebar”]/div/a/strong/text()”)<br /><br /> item_loader.add_css(“topics”, “.zm-tag-editor-labels a::text”)</p><br /> <p> question_item = item_loader.load_item()</p><br /> <p> yield scrapy.Request(self.start_answer_url.format(question_id, 20, 0), headers=self.headers,<br /><br /> callback=self.parse_answer)<br /><br /> yield question_item

def parse_news_metro(self, response):<br /><br /> loader = ItemLoader(item=News(), response=response)<br /><br /> loader.add_value(‘url’, response.url)</p><br /> <p> date_selector = response.css(‘.artikel > div.block-tanggal::text’)<br /><br /> if not date_selector:<br /><br /> return self.parse_news_pilkada(loader, response)<br /><br /> try:<br /><br /> date_time_str = date_selector.extract()[0].split(‘,’)[1].strip()[:-4]<br /><br /> date_time_str=” “.join([_(x) for x in date_time_str.split(‘ ‘)])<br /><br /> published_at_wib = datetime.strptime(date_time_str, ‘%d %B %Y | %H:%M’)<br /><br /> except Exception:<br /><br /> return loader.load_item()<br /><br /> published_at = wib_to_utc(published_at_wib)<br /><br /> if (self.media[‘last_scraped_at’] >= published_at):<br /><br /> is_no_update = True<br /><br /> self.logger.info(‘Media have no update’)<br /><br /> raise CloseSpider(‘finished’)<br /><br /> loader.add_value(‘published_at’, published_at)</p><br /> <p> title_selector = response.css(‘.artikel > h1::text’)<br /><br /> if not title_selector:<br /><br /> return loader.load_item()<br /><br /> loader.add_value(‘title’, title_selector.extract()[0])</p><br /> <p> # Select all p which don’t have iframe inside it<br /><br /> raw_content_selector = response.xpath(‘//div[@class=”artikel”]//p[not(iframe)]’)<br /><br /> if not raw_content_selector:<br /><br /> return loader.load_item()<br /><br /> raw_content=””<br /><br /> for rsl in raw_content_selector:<br /><br /> raw_content = raw_content + rsl.extract().strip()</p><br /> <p> # Go to next page while there is next page button<br /><br /> next_page_selector = response.css(‘.pagination-nb’).xpath(‘//a[text()=”next”]/@href’)<br /><br /> if next_page_selector:<br /><br /> return Request(next_page_selector.extract()[0], callback=lambda x, loader=loader, raw_content=raw_content: self.parse_next_page_metro(x, loader, raw_content))</p><br /> <p> loader.add_value(‘raw_content’, raw_content)</p><br /> <p> # The author usually put inside <strong> tag, however, some news is not using <strong> tag.<br /><br /> # NOTE: this block of code may need revision in the future<br /><br /> author_name=””<br /><br /> for author_name_selector in reversed(raw_content_selector):<br /><br /> author_name_selector = author_name_selector.css(‘strong::text’)<br /><br /> for tmp in reversed(author_name_selector.extract()):<br /><br /> tmp = tmp.strip()<br /><br /> if tmp and all((x.isalpha() and x.isupper()) or x.isspace() or x == ‘.’ or x == ‘|’ for x in tmp):<br /><br /> author_name = tmp<br /><br /> break<br /><br /> if author_name:<br /><br /> break<br /><br /> author_name=”,”.join(author_name.split(‘ | ‘))<br /><br /> loader.add_value(‘author_name’, author_name)<br /><br /> return loader.load_item()

def parse_item(self, response):</p><br /> <p> loader = ItemLoader(EolZhiyeItem(), response)<br /><br /> loader.add_value(‘url’, response.url)<br /><br /> loader.add_value(‘code’, response.url, re=r’/(\w+)\.shtml’)<br /><br /> loader.add_css(‘name’, ‘h1#pagetitle::text’)<br /><br /> loader.add_xpath(‘category’, u’//div[@id=”precontent”]/p[contains(., “??”)]/a/text()’)<br /><br /> loader.add_xpath(‘category2′, u’//div[@id=”precontent”]/p[contains(., “??”)]/a/text()’)<br /><br /> loader.add_xpath(‘detail’, u’//div[@id=”precontent”]/following-sibling::node()[not(self::table)]’, Join(‘\n’))<br /><br /> yield loader.load_item()

def parse_book(self, response):<br /><br /> book_loader = ItemLoader(item=BookItem(), response=response)<br /><br /> book_loader.default_input_processor = MapCompose(remove_tags)<br /><br /> book_loader.default_output_processor = TakeFirst()</p><br /> <p> book_loader.add_xpath(“title”, “//div[@class=”col-sm-6 product_main”]/h1″)<br /><br /> book_loader.add_xpath(“price”, “//p[@class=”price_color”]”)<br /><br /> book_loader.add_xpath(“upc”, “//table[@class=”table table-striped”]/tr[1]/td”)<br /><br /> book_loader.add_xpath(“product_type”, “//table[@class=”table table-striped”]/tr[2]/td”)<br /><br /> book_loader.add_xpath(“tax”, “//table[@class=”table table-striped”]/tr[5]/td”)<br /><br /> book_loader.add_xpath(“stock”, “//table[@class=”table table-striped”]/tr[6]/td”)<br /><br /> book_loader.add_xpath(“reviews”, “//table[@class=”table table-striped”]/tr[7]/td”)<br /><br /> book_loader.add_xpath(“rating”, “//p[@class=”instock availability”]/following-sibling::p/@class”)<br /><br /> yield book_loader.load_item()

def parse_item(self, response):</p><br /> <p> loader = ItemLoader(GaokaopaiZhiyeItem(), response)<br /><br /> loader.add_value(‘url’, response.url)<br /><br /> loader.add_value(‘code’, response.url, re=ur’-([^-]+)\.html’)<br /><br /> loader.add_css(‘name’, u’.modTitle>h1::text’)</p><br /> <p> def parse_category():<br /><br /> for e in response.css(u’.catType>a’):<br /><br /> yield {<br /><br /> ‘url’: e.css(‘::attr(href)’).extract_first(),<br /><br /> ‘code’: e.css(‘::attr(href)’).re_first(ur’-([^-]+)\.html’),<br /><br /> ‘name’: e.css(‘::text’).extract_first(),<br /><br /> }</p><br /> <p> loader.add_value(‘category’, list(parse_category()))<br /><br /> loader.add_css(‘detail’, u’.zhiyeShow’)</p><br /> <p> item = loader.load_item()</p><br /> <p> return FormRequest(<br /><br /> url=”http://www.gaokaopai.com/ajax-career-getRelateMajor.html”,<br /><br /> formdata={‘code’: item[‘code’][0]},<br /><br /> meta={‘item’: item},<br /><br /> dont_filter=True,<br /><br /> callback=self.parse_majors<br /><br /> )

def parse(self, response):<br /><br /> for quote in response.css(“.quote”):<br /><br /> loader = ItemLoader(item=QuoteItem(), selector=quote)</p><br /> <p> loader.add_css(“text”, “.text”)<br /><br /> loader.add_css(“by”, “.authoor”)<br /><br /> loader.add_css(“tags”, “.tag”)<br /><br /> yield loader.load_item()

def parse_question(self, response):<br /><br /> # ??question????????????question item<br /><br /> question_id = response.meta.get(“zhihu_id”, “”)<br /><br /> item_loader = ItemLoader(item=ZhihuQuestionItem(), response=response)<br /><br /> item_loader.add_css(“title”, “h1.QuestionHeader-title::text”)<br /><br /> item_loader.add_css(“content”, “.QuestionHeader-detail”)<br /><br /> item_loader.add_value(“url”, response.url)<br /><br /> item_loader.add_value(“zhihu_id”, question_id)<br /><br /> item_loader.add_css(“answer_num”, “.List-headerText span::text”)<br /><br /> item_loader.add_css(“comments_num”, “.QuestionHeader-actions button::text”)<br /><br /> item_loader.add_css(“watch_user_num”, “.NumberBoard-value::text”)<br /><br /> item_loader.add_css(“topics”, “.QuestionHeader-topics .Popover div::text”)</p><br /> <p> question_item = item_loader.load_item()</p><br /> <p> yield scrapy.Request(self.start_answer_url.format(question_id, 20, 0), headers=self.headers, callback=self.parse_answer)<br /><br /> yield question_item

def _extract_item(self, response):<br /><br /> #?scrapy shell???response<br /><br /> #inspect_response(response, self)</p><br /> <p> #???????scrapy????response?????????????<br /><br /> #open_in_browser(response)</p><br /> <p> #???????<br /><br /> l = ItemLoader(response=response, item=MyspiderItem(), type=”html”)<br /><br /> l.add_xpath(‘movie_name’, ‘//h1/span[@property=”v:itemreviewed”]/text()’)<br /><br /> l.add_xpath(‘movie_year’, ‘//span[@property=”v:initialReleaseDate”]/text()’)<br /><br /> l.add_xpath(‘movie_type’, ‘//span[@property=”v:genre”]/text()’)<br /><br /> l.add_xpath(‘movie_rate’, ‘//strong[@class=”ll rating_num”]/text()’)<br /><br /> l.add_value(‘url’, response.url)<br /><br /> #????????????load_item()????scrapy.Item??<br /><br /> #?scrapy-redis????json?item???????redis?item???<br /><br /> #??json?????python?????????????item?????<br /><br /> return dict(l.load_item())

def parse_first_page(self, response):<br /><br /> count = int(response.xpath(‘//div[@id=”aplist”]/ul/li[1]/a/text()’)[0].re(r’.*?(\d+).*?’)[0])<br /><br /> title = response.request.cookies[‘title’]<br /><br /> albumURL = response.url.replace(“.html”, ”)<br /><br /> for x in xrange(1,count+1):<br /><br /> suffix = “.html”<br /><br /> if x > 1:<br /><br /> suffix = “_”+str(x)+”.html”<br /><br /> request = scrapy.Request(albumURL+suffix, callback=self.parse_item, cookies={‘title’: title})<br /><br /> yield request<br /><br /> l = ItemLoader(item=PageItem(), response=response)<br /><br /> l.add_value(‘title’, title)<br /><br /> l.add_value(‘name’, self.name)<br /><br /> l.add_value(‘url’, response.url)<br /><br /> l.add_xpath(‘image_urls’, ‘//p[@id=”contents”]/a/img/@src’)<br /><br /> yield l.load_item()

def parse_first_page(self, response):<br /><br /> count = int(response.xpath(‘//ul[@class=”image”]/text()’)[0].re(r’.*?(\d+).*?’)[0])<br /><br /> title = response.request.cookies[‘title’]<br /><br /> albumURL = response.url.replace(“.shtml”, ”)<br /><br /> # print u”, count, title, albumURL<br /><br /> for x in xrange(1,count+1):<br /><br /> suffix = “.shtml”<br /><br /> if x > 1:<br /><br /> suffix = “_”+str(x)+”.shtml”<br /><br /> # print u”,albumURL+suffix<br /><br /> request = scrapy.Request(albumURL+suffix, callback=self.parse_item, cookies={‘title’: title})<br /><br /> yield request</p><br /> <p> l = ItemLoader(item=PageItem(), response=response)<br /><br /> l.add_value(‘title’, title)<br /><br /> l.add_value(‘name’, self.name)<br /><br /> l.add_value(‘url’, response.url)<br /><br /> l.add_xpath(‘image_urls’, ‘//td[@valign=”top”]/img/@src’)<br /><br /> yield l.load_item()