sudo pip install virtualenv #安装虚拟环境工具
virtualenv ENV #创建一个虚拟环境目录
source ./ENV/bin/active #激活虚拟环境
pip install Scrapy
#验证是否安装成功
pip list

#输出如下
cffi (0.8.6)
cryptography (0.6.1)
cssselect (0.9.1)
lxml (3.4.1)
pip (1.5.6)
pycparser (2.10)
pyOpenSSL (0.14)
queuelib (1.2.2)
Scrapy (0.24.4)
setuptools (3.6)
six (1.8.0)
Twisted (14.0.2)
w3lib (1.10.0)
wsgiref (0.1.2)
zope.interface (4.1.1)

$ scrapy startproject tutorial

.
├── scrapy.cfg
└── tutorial
 ├── __init__.py
 ├── items.py
 ├── pipelines.py
 ├── settings.py
 └── spiders
  └── __init__.py

from scrapy.item import Item, Field


class DmozItem(Item):
 # define the fields for your item here like:
 name = Field()
 description = Field()
 url = Field()

import scrapy

class DmozSpider(scrapy.Spider):
 name = "dmoz"
 allowed_domains = ["dmoz.org"]
 start_urls = [
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
 ]

 def parse(self, response):
  filename = response.url.split("/")[-2]
  with open(filename, 'wb') as f:
   f.write(response.body)

├── scrapy.cfg
└── tutorial
 ├── __init__.py
 ├── items.py
 ├── pipelines.py
 ├── settings.py
 └── spiders
  ├── __init__.py
  └── dmoz_spider.py

$ scrapy crawl dmoz

2014-12-15 09:30:59+0800 [scrapy] INFO: Scrapy 0.24.4 started (bot: tutorial)
2014-12-15 09:30:59+0800 [scrapy] INFO: Optional features available: ssl, http11
2014-12-15 09:30:59+0800 [scrapy] INFO: Overridden settings: {'NEWSPIDER_MODULE': 'tutorial.spiders', 'SPIDER_MODULES': ['tutorial.spiders'], 'BOT_NAME': 'tutorial'}
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled extensions: LogStats, TelnetConsole, CloseSpider, WebService, CoreStats, SpiderState
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled downloader middlewares: HttpAuthMiddleware, DownloadTimeoutMiddleware, UserAgentMiddleware, RetryMiddleware, DefaultHeadersMiddleware, MetaRefreshMiddleware, HttpCompressionMiddleware, RedirectMiddleware, CookiesMiddleware, ChunkedTransferMiddleware, DownloaderStats
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled spider middlewares: HttpErrorMiddleware, OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled item pipelines:
2014-12-15 09:30:59+0800 [dmoz] INFO: Spider opened
2014-12-15 09:30:59+0800 [dmoz] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2014-12-15 09:30:59+0800 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023
2014-12-15 09:30:59+0800 [scrapy] DEBUG: Web service listening on 127.0.0.1:6080
2014-12-15 09:31:00+0800 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/> (referer: None)
2014-12-15 09:31:00+0800 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
2014-12-15 09:31:00+0800 [dmoz] INFO: Closing spider (finished)
2014-12-15 09:31:00+0800 [dmoz] INFO: Dumping Scrapy stats:
 {'downloader/request_bytes': 516,
  'downloader/request_count': 2,
  'downloader/request_method_count/GET': 2,
  'downloader/response_bytes': 16338,
  'downloader/response_count': 2,
  'downloader/response_status_count/200': 2,
  'finish_reason': 'finished',
  'finish_time': datetime.datetime(2014, 12, 15, 1, 31, 0, 666214),
  'log_count/DEBUG': 4,
  'log_count/INFO': 7,
  'response_received_count': 2,
  'scheduler/dequeued': 2,
  'scheduler/dequeued/memory': 2,
  'scheduler/enqueued': 2,
  'scheduler/enqueued/memory': 2,
  'start_time': datetime.datetime(2014, 12, 15, 1, 30, 59, 533207)}
2014-12-15 09:31:00+0800 [dmoz] INFO: Spider closed (finished)

<ul class="directory-url" style="margin-left:0;">

 <li><a href="http://www.pearsonhighered.com/educator/academic/product/0,,0130260363,00%2Ben-USS_01DBC.html" class="listinglink">Core Python Programming</a> 
- By Wesley J. Chun; Prentice Hall PTR, 2001, ISBN 0130260363. For experienced developers to improve extant skills; professional level examples. Starts by introducing syntax, objects, error handling, functions, classes, built-ins. [Prentice Hall]
<div class="flag"><a href="/public/flag?cat=Computers%2FProgramming%2FLanguages%2FPython%2FBooks&url=http%3A%2F%2Fwww.pearsonhighered.com%2Feducator%2Facademic%2Fproduct%2F0%2C%2C0130260363%2C00%252Ben-USS_01DBC.html"><img src="/img/flag.png" alt="[!]" title="report an issue with this listing"></a></div>
</li>
...省略部分...
</ul>

#通过如下命令选择每个在网站中的 <li> 元素:
sel.xpath('//ul/li')

#网站描述:
sel.xpath('//ul/li/text()').extract()

#网站标题:
sel.xpath('//ul/li/a/text()').extract()

#网站链接:
sel.xpath('//ul/li/a/@href').extract()

for sel in response.xpath('//ul/li')
 title = sel.xpath('a/text()').extract()
 link = sel.xpath('a/@href').extract()
 desc = sel.xpath('text()').extract()
 print title, link, desc

import scrapy

class DmozSpider(scrapy.Spider):
 name = "dmoz"
 allowed_domains = ["dmoz.org"]
 start_urls = [
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
 ]

 def parse(self, response):
  for sel in response.xpath('//ul/li'):
   title = sel.xpath('a/text()').extract()
   link = sel.xpath('a/@href').extract()
   desc = sel.xpath('text()').extract()
   print title, link, desc

>>> item = DmozItem()
>>> item['title'] = 'Example title'
>>> item['title']
'Example title'

from scrapy.spider import Spider
from scrapy.selector import Selector
from tutorial.items import DmozItem


class DmozSpider(Spider):
 name = "dmoz"
 allowed_domains = ["dmoz.org"]
 start_urls = [
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
 ]

 def parse(self, response):
  sel = Selector(response)
  sites = sel.xpath('//ul[@class="directory-url"]/li')
  items = []

  for site in sites:
   item = DmozItem()
   item['name'] = site.xpath('a/text()').extract()
   item['url'] = site.xpath('a/@href').extract()
   item['description'] = site.xpath('text()').re('-\s[^\n]*\\r')
   items.append(item)
  return items

from scrapy.exceptions import DropItem

class TutorialPipeline(object):

 # put all words in lowercase
 words_to_filter = ['politics', 'religion']

 def process_item(self, item, spider):
  for word in self.words_to_filter:
   if word in unicode(item['description']).lower():
    raise DropItem("Contains forbidden word: %s" % word)
  else:
   return item

ITEM_PIPELINES = {'tutorial.pipelines.FilterWordsPipeline': 1}

class Spider(object_ref):
 """Base class for scrapy spiders. All spiders must inherit from this
 class.
 """
 
 name = None
 
 def __init__(self, name=None, **kwargs):
  if name is not None:
   self.name = name
  elif not getattr(self, 'name', None):
   raise ValueError("%s must have a name" % type(self).__name__)
  self.__dict__.update(kwargs)
  if not hasattr(self, 'start_urls'):
   self.start_urls = []
 
 def start_requests(self):
  for url in self.start_urls:
   yield self.make_requests_from_url(url)
 
 def make_requests_from_url(self, url):
  return Request(url, dont_filter=True)
 
 def parse(self, response):
  raise NotImplementedError
 
 
BaseSpider = create_deprecated_class('BaseSpider', Spider)

import scrapyfrom myproject.items import MyItemclass MySpider(scrapy.Spider):
 name = 'example.com'
 allowed_domains = ['example.com']
 start_urls = [
  'http://www.example.com/1.html',
  'http://www.example.com/2.html',
  'http://www.example.com/3.html',
 ]
 
 def parse(self, response):
  sel = scrapy.Selector(response)
  for h3 in response.xpath('//h3').extract():
   yield MyItem(title=h3)
 
  for url in response.xpath('//a/@href').extract():
   yield scrapy.Request(url, callback=self.parse)

class MininovaSpider(CrawlSpider):
 name = 'mininova'
 allowed_domains = ['mininova.org']
 start_urls = ['http://www.mininova.org/yesterday']
 rules = [Rule(LinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]
 
 def parse_torrent(self, response):
  torrent = TorrentItem()
  torrent['url'] = response.url
  torrent['name'] = response.xpath("//h1/text()").extract()
  torrent['description'] = response.xpath("//div[@id='description']").extract()
  torrent['size'] = response.xpath("//div[@id='specifications']/p[2]/text()[2]").extract()
  return torrent

 rules = (
  # 提取匹配 'category.php' (但不匹配 'subsection.php') 的链接并跟进链接(没有callback意味着follow默认为True)
  Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
 
  # 提取匹配 'item.php' 的链接并使用spider的parse_item方法进行分析
  Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
 )