#!/usr/bin/py2
# -*- coding: utf-8 -*-
#encoding=utf-8

from bs4 import BeautifulSoup
from scrapy.http import Request, FormRequest
from spider_test.items import *
from scrapy.spiders import CrawlSpider
from spider_test import settings

class ScrapyTestSpider(CrawlSpider):

    name = "spider_test"
    allowed_domains = [settings.SPIDER_DOMAIN]　　　　# 這是在settings.py文件中的自定義配置，其值類似于 www.demo.com

    def start_requests(self):
        """第一次請求一下登錄頁面，設置開啟cookie使其得到cookie，設置回調函數"""
        yield Request('http://%s/admin/account/login.html' % settings.SPIDER_DOMAIN, meta={'cookiejar': 1}, callback=self.parse)

    def parse(self, response):
        data = dict(username="xiaoming",    # 登錄頁表單的賬號字段
                    password="888888")        # 登錄頁表單的密碼字段

        print('登錄中....!')
        """第二次用表單post請求，攜帶Cookie、瀏覽器代理、用戶登錄信息，進行登錄給Cookie授權"""
        yield FormRequest(url='http://%s/admin/account/dologin.html' % settings.SPIDER_DOMAIN,  # 真實post地址
                          meta={'cookiejar': 1},
                          formdata=data,
                          callback=self.jump_list)

    def jump_list(self, response):
        print('正在請需要登錄才可以訪問的頁面....!')
        yield Request('http://%s/admin/office/getofficelist.html' % settings.SPIDER_DOMAIN,
                      meta={'cookiejar': 1}, callback=self.parser_list)

    def parser_list(self, response):
        soup = BeautifulSoup(response.body, 'html.parser')
        page_list = soup.find(attrs={'class': 'pagination'}).find_all('a')
        if page_list:
            for page in page_list:
                page_url = 'http://%s%s' % (settings.SPIDER_DOMAIN, page.get('href'))
                yield Request(page_url, meta={'cookiejar': 1}, callback=self.parser_list)

        office_list = soup.find_all('a', attrs={'class': 'ui-office-list'})
        if office_list:
            for office in office_list:
                office_url = 'http://%s%s' % (settings.SPIDER_DOMAIN, office.attrs['href'])
                yield Request(office_url, meta={'cookiejar': 1}, callback=self.parse_article)

    def parse_article(self, response):
        test_item = SpiderTestItem()
        soup = BeautifulSoup(response.body, 'html.parser')
        container = soup.find('table', attrs={'class': 'index-statistics-table'})
        test_item['source_url'] = response.url
        test_item['title'] = soup.title.get_text()
        test_item['article_content'] = container.prettify()
        return test_item

#!/usr/bin/py2
# -*- coding: utf-8 -*-
#encoding=utf-8

from bs4 import BeautifulSoup
from scrapy.http import Request, FormRequest
from spider_test.items import *
from scrapy.spiders import CrawlSpider
from spider_test import settings

class ScrapyTestSpider(CrawlSpider):

    name = "spider_test"
    allowed_domains = [settings.SPIDER_DOMAIN]

    cookies = dict(PHPSESSID='qwertyuiopasdfghjklzxcvbnm')　　# 登錄后得到的會話ID

    def start_requests(self):
        print('正在請需要登錄才可以訪問的頁面....!')
        yield Request('http://%s/admin/office/getofficelist.html' % settings.SPIDER_DOMAIN, cookies=self.cookies,
                    callback=self.parser_list)

    def parser_list(self, response):
        soup = BeautifulSoup(response.body, 'html.parser')
        page_list = soup.find(attrs={'class': 'pagination'}).find_all('a')
        if page_list:
            for page in page_list:
                page_url = 'http://%s%s' % (settings.SPIDER_DOMAIN, page.get('href'))
                yield Request(page_url, cookies=self.cookies, callback=self.parser_list)

        office_list = soup.find_all('a', attrs={'class': 'ui-office-list'})
        if office_list:
            for office in office_list:
                office_url = 'http://%s%s' % (settings.SPIDER_DOMAIN, office.attrs['href'])
                yield Request(office_url, cookies=self.cookies, callback=self.parse_article)

    def parse_article(self, response):
        test_item = SpiderTestItem()
        soup = BeautifulSoup(response.body, 'html.parser')
        container = soup.find('table', attrs={'class': 'index-statistics-table'})
        test_item['source_url'] = response.url
        test_item['title'] = soup.title.get_text()
        test_item['article_content'] = container.prettify()
        return test_item