mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-23 22:44:03 +00:00
improved Scrapy documentation index for better usability
This commit is contained in:
Pablo Hoffman
parent
af6db1691e
commit
0c4c153819
39
docs/_templates/index.html
vendored
Normal file
39
docs/_templates/index.html
vendored
Normal file
@ -0,0 +1,39 @@
|
||||
{% extends "defindex.html" %}
|
||||
{% block tables %}
|
||||
<p><strong>Parts of the documentation:</strong></p>
|
||||
|
||||
<table class="contentstable" align="center"><tr>
|
||||
<td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("intro/index") }}">Getting started</a><br/>
|
||||
<span class="linkdescr">for an overview and tutorial</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("ref/index") }}">API Reference</a><br/>
|
||||
<span class="linkdescr">all details about Scrapy stable API</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("howto/index") }}">Experimental Features</a><br/>
|
||||
<span class="linkdescr">use with caution</span></p>
|
||||
</td><td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("topics/index") }}">Using Scrapy</a><br/>
|
||||
<span class="linkdescr">usage guide and key concepts</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("faq") }}">Frequently Asked Questions</a><br/>
|
||||
<span class="linkdescr">for common questions</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("proposed/index") }}">Proposed documentation</a><br/>
|
||||
<span class="linkdescr">documentation in progress or unstable features</span></p>
|
||||
</td></tr>
|
||||
</table>
|
||||
|
||||
<p><strong>Indices and tables:</strong></p>
|
||||
<table class="contentstable" align="center"><tr>
|
||||
<td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("modindex") }}">Global Module Index</a><br/>
|
||||
<span class="linkdescr">quick access to all modules</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("genindex") }}">General Index</a><br/>
|
||||
<span class="linkdescr">all functions, classes, terms</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("glossary") }}">Glossary</a><br/>
|
||||
<span class="linkdescr">the most important terms explained</span></p>
|
||||
</td><td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("search") }}">Search page</a><br/>
|
||||
<span class="linkdescr">search this documentation</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("contents") }}">Complete Table of Contents</a><br/>
|
||||
<span class="linkdescr">lists all sections and subsections</span></p>
|
||||
</td></tr>
|
||||
</table>
|
||||
{% endblock %}
|
||||
16
docs/_templates/layout.html
vendored
Normal file
16
docs/_templates/layout.html
vendored
Normal file
@ -0,0 +1,16 @@
|
||||
{% extends "!layout.html" %}
|
||||
|
||||
{% block rootrellink %}
|
||||
Scrapy v. {{ version }} documentation »
|
||||
<li><a href="{{ pathto('index') }}">Home</a> | </li>
|
||||
<li><a href="{{ pathto('intro/index') }}">Getting Started</a> | </li>
|
||||
<li><a href="{{ pathto('topics/index') }}">Using Scrapy</a> | </li>
|
||||
<li><a href="{{ pathto('ref/index') }}">API reference</a> | </li>
|
||||
<li><a href="{{ pathto('faq') }}">FAQ</a> | </li>
|
||||
<li><a href="{{ pathto('search') }}">Search</a> </li>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
|
||||
|
||||
|
||||
10
docs/conf.py
10
docs/conf.py
@ -11,7 +11,7 @@
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
import sys, os
|
||||
import sys
|
||||
from os import path
|
||||
|
||||
# If your extensions are in another directory, add it here. If the directory
|
||||
@ -29,7 +29,7 @@ sys.path.append(path.join(path.dirname(path.dirname(__file__)), "scrapy"))
|
||||
extensions = ['scrapydocs', 'sphinx.ext.autodoc']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['../templates/docs']
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix of source filenames.
|
||||
source_suffix = '.rst'
|
||||
@ -38,7 +38,7 @@ source_suffix = '.rst'
|
||||
#source_encoding = 'utf-8'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
master_doc = 'contents'
|
||||
|
||||
# General information about the project.
|
||||
project = u'Scrapy'
|
||||
@ -130,7 +130,9 @@ html_use_smartypants = True
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
#html_additional_pages = {}
|
||||
html_additional_pages = {
|
||||
'index': 'index.html'
|
||||
}
|
||||
|
||||
# If false, no module index is generated.
|
||||
#html_use_modindex = True
|
||||
|
||||
15
docs/contents.rst
Normal file
15
docs/contents.rst
Normal file
@ -0,0 +1,15 @@
|
||||
.. _contents:
|
||||
|
||||
=============================
|
||||
Scrapy documentation contents
|
||||
=============================
|
||||
|
||||
.. toctree::
|
||||
|
||||
intro/index
|
||||
topics/index
|
||||
ref/index
|
||||
faq
|
||||
topics/adaptors
|
||||
experimental
|
||||
proposed/index
|
||||
15
docs/experimental.rst
Normal file
15
docs/experimental.rst
Normal file
@ -0,0 +1,15 @@
|
||||
.. _ref-experimental:
|
||||
|
||||
Experimental features
|
||||
=====================
|
||||
|
||||
This section documents experimental features that may become stable in future
|
||||
Scrapy releases, but whose API is not yet stable. Use them with caution, and
|
||||
subscribe to the `mailing lists <http://scrapy.org/community/>`_ to be notified
|
||||
of any changes.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
topics/adaptors
|
||||
|
||||
@ -29,8 +29,8 @@ Does Scrapy work with Python 3.0?
|
||||
No, and there are no plans to port Scrapy to Python 3.0 yet. At the moment
|
||||
Scrapy works with Python 2.5 or 2.6.
|
||||
|
||||
Does Scrapy "stole" X from Django?
|
||||
----------------------------------
|
||||
Did Scrapy "steal" X from Django?
|
||||
---------------------------------
|
||||
|
||||
Probably, but we don't like that word. We think Django_ is a great open source
|
||||
project and an example to follow, so we've used it as an inspiration for
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
.. _index:
|
||||
|
||||
==============================
|
||||
Scrapy |version| documentation
|
||||
==============================
|
||||
|
||||
Welcome! This is the documentation for Scrapy, a screen scraping framework for
|
||||
Python.
|
||||
|
||||
For more information about the project visit the `Scrapy homepage
|
||||
<http://scrapy.org>`_.
|
||||
|
||||
Getting started
|
||||
===============
|
||||
|
||||
This section helps you get familiarized with Scrapy and its basic concepts.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
intro/index
|
||||
|
||||
Using Scrapy
|
||||
============
|
||||
|
||||
This section explains all key concepts of Scrapy.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
topics/index
|
||||
|
||||
API reference
|
||||
=============
|
||||
|
||||
This section documents the :ref:`Scrapy API <ref-index>`. See also
|
||||
:ref:`misc-api-stability`.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
ref/index
|
||||
|
||||
Frequently asked questions
|
||||
==========================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
faq
|
||||
|
||||
Experimental features
|
||||
=====================
|
||||
|
||||
This section documents experimental features that may become stable in future
|
||||
Scrapy releases, but whose API is not yet stable. Use them with caution, and
|
||||
subscribe to the `mailing lists <http://scrapy.org/community/>`_ to be notified
|
||||
of any changes.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
topics/adaptors
|
||||
|
||||
Proposed documentation
|
||||
======================
|
||||
|
||||
This section contains documentation which is not yet completed, or which
|
||||
overlaps with existing documentation and needs to be manually merged. It may
|
||||
also contain outdated information, as it's not revised so frequently.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
proposed/index
|
||||
@ -20,7 +20,7 @@ so you can get an idea of how it works and decide if Scrapy is what you need.
|
||||
|
||||
When you're ready to start a project, you can :ref:`start with the tutorial
|
||||
<intro-tutorial>`. For more detailed information you can take a look at the
|
||||
:ref:`documentation contents <index>`.
|
||||
:ref:`documentation contents <contents>`.
|
||||
|
||||
Pick a website
|
||||
==============
|
||||
|
||||
@ -3,6 +3,10 @@
|
||||
Proposed documentation
|
||||
======================
|
||||
|
||||
This section contains documentation which is not yet completed, or which
|
||||
overlaps with existing documentation and needs to be manually merged. It may
|
||||
also contain outdated information, as it's not revised so frequently.
|
||||
|
||||
.. warning::
|
||||
|
||||
This is experimental or "work in progress" documentation, use at your own
|
||||
@ -14,7 +18,4 @@ Proposed documentation
|
||||
newitem
|
||||
spiders
|
||||
|
||||
.. toctree::
|
||||
|
||||
introduction
|
||||
|
||||
|
||||
@ -1,49 +0,0 @@
|
||||
============
|
||||
Introduction
|
||||
============
|
||||
|
||||
.. architecture:
|
||||
|
||||
Overview
|
||||
========
|
||||
|
||||
.. image:: _images/scrapy_architecture.png
|
||||
:width: 700
|
||||
:height: 468
|
||||
:alt: Scrapy architecture
|
||||
|
||||
Requests and Responses
|
||||
----------------------
|
||||
|
||||
Scrapy uses *Requests* and *Responses* for crawling web sites.
|
||||
|
||||
Generally, *Requests* are generated in the Spiders and pass across the system
|
||||
until they reach the *Downloader*, which executes the *Request* and returns a
|
||||
*Response* which goes back to the Spider that generated the *Request*.
|
||||
|
||||
Spiders
|
||||
-------
|
||||
|
||||
Spiders are user written classes to scrape information from a domain (or group
|
||||
of domains).
|
||||
|
||||
They define an initial set of URLs (or Requests) to download, how to crawl the
|
||||
domain and how to scrape *Items* from their pages.
|
||||
|
||||
Items
|
||||
-----
|
||||
|
||||
Items are the placeholder to use for the scraped data. They are represented by a
|
||||
simple Python class.
|
||||
|
||||
After an Item has been scraped by a Spider, it is sent to the Item Pipeline for further proccesing.
|
||||
|
||||
Item Pipeline
|
||||
-------------
|
||||
|
||||
The Item Pipeline is a list of user written Python classes that implement a
|
||||
specific method, which is called sequentially for every element of the Pipeline.
|
||||
|
||||
Each element receives the Scraped Item, do an action upon it (like validating,
|
||||
checking for duplicates, store the item), and then decide if the Item continues
|
||||
trough the Pipeline or the item is dropped.
|
||||
@ -6,16 +6,17 @@ API Reference
|
||||
This section documents the Scrapy |version| API. For more information see :ref:`misc-api-stability`.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
request-response
|
||||
spiders
|
||||
selectors
|
||||
settings
|
||||
signals
|
||||
exceptions
|
||||
request-response
|
||||
logging
|
||||
email
|
||||
extension-manager
|
||||
extensions
|
||||
downloader-middleware
|
||||
settings
|
||||
signals
|
||||
logging
|
||||
email
|
||||
link-extractors
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user