mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-23 14:24:19 +00:00
rearranged documentation into a better organization
--HG-- rename : docs/topics/index.rst => docs/index.rst
This commit is contained in:
Pablo Hoffman
parent
4761e0c88d
commit
9635a7839c
37
docs/_templates/index.html
vendored
37
docs/_templates/index.html
vendored
@ -1,37 +0,0 @@
|
||||
{% extends "defindex.html" %}
|
||||
{% block tables %}
|
||||
<p><strong>Parts of the documentation:</strong></p>
|
||||
|
||||
<table class="contentstable" align="center"><tr>
|
||||
<td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("intro/index") }}">Getting started</a><br/>
|
||||
<span class="linkdescr">for an overview and tutorial</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("topics/index") }}">Using Scrapy</a><br/>
|
||||
<span class="linkdescr">usage guide and key concepts</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("reference") }}">API Reference</a><br/>
|
||||
<span class="linkdescr">all details about Scrapy stable API</span></p>
|
||||
</td><td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("faq") }}">Frequently Asked Questions</a><br/>
|
||||
<span class="linkdescr">for common questions</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("experimental/index") }}">Experimental Features</a><br/>
|
||||
<span class="linkdescr">use with caution</span></p>
|
||||
</td></tr>
|
||||
</table>
|
||||
|
||||
<p><strong>Indices and tables:</strong></p>
|
||||
<table class="contentstable" align="center"><tr>
|
||||
<td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("modindex") }}">Global Module Index</a><br/>
|
||||
<span class="linkdescr">quick access to all modules</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("genindex") }}">General Index</a><br/>
|
||||
<span class="linkdescr">all functions, classes, terms</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("glossary") }}">Glossary</a><br/>
|
||||
<span class="linkdescr">the most important terms explained</span></p>
|
||||
</td><td width="50%">
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("search") }}">Search page</a><br/>
|
||||
<span class="linkdescr">search this documentation</span></p>
|
||||
<p class="biglink"><a class="biglink" href="{{ pathto("contents") }}">Complete Table of Contents</a><br/>
|
||||
<span class="linkdescr">lists all sections and subsections</span></p>
|
||||
</td></tr>
|
||||
</table>
|
||||
{% endblock %}
|
||||
9
docs/_templates/layout.html
vendored
9
docs/_templates/layout.html
vendored
@ -1,9 +0,0 @@
|
||||
{% extends "!layout.html" %}
|
||||
|
||||
{% block rootrellink %}
|
||||
<li><a href="{{ pathto('index') }}">{{ shorttitle|e }}</a> (</li>
|
||||
<li><a title="Introduction and tutorial" href="{{ pathto('intro/index') }}">intro</a> | </li>
|
||||
<li><a title="Usage guide" href="{{ pathto('topics/index') }}">topics</a> | </li>
|
||||
<li><a title="API Reference" href="{{ pathto('reference') }}">reference</a> | </li>
|
||||
<li><a title="Frequently Asked Questions" href="{{ pathto('faq') }}">faq</a> ) {{ reldelim1 }} </li>
|
||||
{% endblock %}
|
||||
@ -38,7 +38,7 @@ source_suffix = '.rst'
|
||||
#source_encoding = 'utf-8'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'contents'
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'Scrapy'
|
||||
@ -130,9 +130,7 @@ html_use_smartypants = True
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
html_additional_pages = {
|
||||
'index': 'index.html'
|
||||
}
|
||||
#html_additional_pages = {}
|
||||
|
||||
# If false, no module index is generated.
|
||||
#html_use_modindex = True
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
.. _contents:
|
||||
|
||||
=============================
|
||||
Scrapy documentation contents
|
||||
=============================
|
||||
|
||||
.. toctree::
|
||||
|
||||
intro/index
|
||||
topics/index
|
||||
faq
|
||||
experimental/index
|
||||
reference
|
||||
api-stability
|
||||
@ -1,4 +1,4 @@
|
||||
.. _experimental-index:
|
||||
.. _experimental:
|
||||
|
||||
Experimental features
|
||||
=====================
|
||||
|
||||
198
docs/index.rst
Normal file
198
docs/index.rst
Normal file
@ -0,0 +1,198 @@
|
||||
.. _topics-index:
|
||||
|
||||
====================
|
||||
Scrapy documentation
|
||||
====================
|
||||
|
||||
This documentation contains everything you need to know about Scrapy.
|
||||
|
||||
Getting help
|
||||
============
|
||||
|
||||
Having trouble? We'd like to help!
|
||||
|
||||
* Try the :ref:`FAQ <faq>` -- it's got answers to some common questions.
|
||||
* Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`.
|
||||
* Search for information in the `archives of the scrapy-users mailing list`_, or
|
||||
`post a question`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_
|
||||
* Report bugs with Scrapy in our `ticket tracker`_.
|
||||
|
||||
.. _archives of the scrapy-users mailing list: http://groups.google.com/group/scrapy-users/
|
||||
.. _post a question: http://groups.google.com/group/scrapy-users/
|
||||
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
|
||||
.. _ticket tracker: http://dev.scrapy.org/
|
||||
|
||||
|
||||
First steps
|
||||
===========
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
intro/overview
|
||||
intro/install
|
||||
intro/tutorial
|
||||
|
||||
:ref:`intro-overview`
|
||||
Understand what Scrapy is and how it can help you.
|
||||
|
||||
:ref:`intro-install`
|
||||
Get Scrapy installed on your computer.
|
||||
|
||||
:ref:`intro-tutorial`
|
||||
Write your first Scrapy project
|
||||
|
||||
|
||||
Scraping basics
|
||||
===============
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
topics/items
|
||||
topics/spiders
|
||||
topics/link-extractors
|
||||
topics/selectors
|
||||
topics/loaders
|
||||
topics/shell
|
||||
topics/item-pipeline
|
||||
|
||||
:ref:`topics-items`
|
||||
Define the data you want to scrape.
|
||||
|
||||
:ref:`topics-spiders`
|
||||
Write the rules to crawl your websites.
|
||||
|
||||
:ref:`topics-selectors`
|
||||
Extract the data from web pages.
|
||||
|
||||
:ref:`topics-loaders`
|
||||
Populate your items with the extracted data.
|
||||
|
||||
:ref:`topics-shell`
|
||||
Test your extracted data in an interactive environment.
|
||||
|
||||
:ref:`topics-item-pipeline`
|
||||
Post-process and store your scraped data.
|
||||
|
||||
|
||||
Built-in services
|
||||
=================
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
topics/logging
|
||||
topics/stats
|
||||
topics/email
|
||||
topics/telnetconsole
|
||||
topics/webconsole
|
||||
|
||||
:ref:`topics-logging`
|
||||
Understand the simple logging facility provided by Scrapy.
|
||||
|
||||
:ref:`topics-stats`
|
||||
Collect statistics about your scraping crawler.
|
||||
|
||||
:ref:`topics-email`
|
||||
Send email notifications when certain events occur.
|
||||
|
||||
:ref:`topics-telnetconsole`
|
||||
Inspect a running crawler using a built-in Python console.
|
||||
|
||||
:ref:`topics-webconsole`
|
||||
Monitor and control a crawler using a web interface.
|
||||
|
||||
|
||||
Solving specific problems
|
||||
=========================
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
faq
|
||||
topics/firefox
|
||||
topics/firebug
|
||||
topics/leaks
|
||||
|
||||
:ref:`faq`
|
||||
Get answers to most frequently asked questions.
|
||||
|
||||
:ref:`topics-firefox`
|
||||
Learn how to scrape with Firefox and some useful add-ons.
|
||||
|
||||
:ref:`topics-firebug`
|
||||
Learn how to scrape efficiently using Firebug.
|
||||
|
||||
:ref:`topics-leaks`
|
||||
Learn how to find and get rid of memory leaks in your crawler.
|
||||
|
||||
|
||||
Extending Scrapy
|
||||
================
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
topics/architecture
|
||||
topics/downloader-middleware
|
||||
topics/spider-middleware
|
||||
topics/scheduler-middleware
|
||||
topics/extensions
|
||||
|
||||
:ref:`topics-architecture`
|
||||
Understand the Scrapy architecture.
|
||||
|
||||
:ref:`topics-downloader-middleware`
|
||||
Customize how pages get requested and downloaded.
|
||||
|
||||
:ref:`topics-spider-middleware`
|
||||
Customize the input and output of your spiders.
|
||||
|
||||
:ref:`topics-scheduler-middleware`
|
||||
Customize how pages are scheduled.
|
||||
|
||||
:ref:`topics-extensions`
|
||||
Add any custom functionality using :ref:`signals <topics-signals>` and the
|
||||
Scrapy API
|
||||
|
||||
|
||||
Reference
|
||||
=========
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
topics/request-response
|
||||
topics/settings
|
||||
topics/signals
|
||||
topics/exceptions
|
||||
|
||||
:ref:`topics-request-response`
|
||||
Understand the classes used to represent HTTP requests and responses.
|
||||
|
||||
:ref:`topics-settings`
|
||||
Learn how to configure Scrapy and see all :ref:`available settings <topics-settings-ref>`.
|
||||
|
||||
:ref:`topics-signals`
|
||||
See all available signals and how to work with them.
|
||||
|
||||
:ref:`topics-exceptions`
|
||||
See all available exceptions and their meaning.
|
||||
|
||||
|
||||
All the rest
|
||||
============
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
api-stability
|
||||
experimental/index
|
||||
|
||||
:ref:`api-stability`
|
||||
Understand Scrapy versioning and API stability
|
||||
|
||||
:ref:`experimental`
|
||||
Learn about bleeding-edge features
|
||||
@ -1,32 +0,0 @@
|
||||
.. _intro:
|
||||
|
||||
Getting started
|
||||
===============
|
||||
|
||||
If you're new to Scrapy, web crawling or screen scraping this chapter will get
|
||||
you started quickly by giving you an overview of what Scrapy is and how it
|
||||
might help you.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
overview
|
||||
install
|
||||
tutorial
|
||||
|
||||
.. seealso::
|
||||
|
||||
If you're new to Python_, you might want to start by getting an idea of what
|
||||
the language is like, to get the most out of Scrapy (since Scrapy is 100%
|
||||
Python).
|
||||
|
||||
If you're new to programming and want to start with Python, take a look at
|
||||
`this list of Python resources for non-programmers`_.
|
||||
|
||||
If you're already familiar with other languages, and want to learn Python
|
||||
quickly, we recommend `Dive Into Python`_.
|
||||
|
||||
.. _Python: http://www.python.org
|
||||
.. _this list of Python resources for non-programmers: http://wiki.python.org/moin/BeginnersGuide/NonProgrammers
|
||||
.. _Dive Into Python: http://www.diveintopython.org
|
||||
|
||||
@ -19,8 +19,7 @@ The purpose of this document is to introduce you to the concepts behind Scrapy
|
||||
so you can get an idea of how it works and decide if Scrapy is what you need.
|
||||
|
||||
When you're ready to start a project, you can :ref:`start with the tutorial
|
||||
<intro-tutorial>`. For more detailed information you can take a look at the
|
||||
:ref:`documentation contents <contents>`.
|
||||
<intro-tutorial>`.
|
||||
|
||||
Pick a website
|
||||
==============
|
||||
|
||||
@ -19,6 +19,17 @@ This tutorial will walk you through through these tasks:
|
||||
4. Writing an :ref:`Item Pipeline <topics-item-pipeline>` to store the
|
||||
extracted Items
|
||||
|
||||
Scrapy is written in Python_. If you're new to the language you might want to
|
||||
start by getting an idea of what the language is like, to get the most out of
|
||||
Scrapy. If you're already familiar with other languages, and want to learn
|
||||
Python quickly, we recommend `Dive Into Python`_. If you're new to programming
|
||||
and want to start with Python, take a look at `this list of Python resources
|
||||
for non-programmers`_.
|
||||
|
||||
.. _Python: http://www.python.org
|
||||
.. _this list of Python resources for non-programmers: http://wiki.python.org/moin/BeginnersGuide/NonProgrammers
|
||||
.. _Dive Into Python: http://www.diveintopython.org
|
||||
|
||||
Creating a project
|
||||
==================
|
||||
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
.. _ref:
|
||||
|
||||
API Reference
|
||||
=============
|
||||
|
||||
This section documents the Scrapy |version| API. For more information see :ref:`api-stability`.
|
||||
|
||||
* :ref:`topics-request-response`
|
||||
* :ref:`topics-spiders-ref`
|
||||
* :ref:`topics-selectors-ref`
|
||||
* :ref:`topics-settings-ref`
|
||||
* :ref:`topics-signals-ref`
|
||||
* :ref:`topics-exceptions-ref`
|
||||
* :ref:`topics-logging`
|
||||
* :ref:`topics-email`
|
||||
* :ref:`topics-extensions-ref`
|
||||
* :ref:`topics-downloader-middleware-ref`
|
||||
* :ref:`topics-spider-middleware-ref`
|
||||
* :ref:`topics-scheduler-middleware-ref`
|
||||
* :ref:`topics-link-extractors-ref`
|
||||
* :ref:`topics-stats-ref`
|
||||
|
||||
@ -1,76 +0,0 @@
|
||||
.. _topics-index:
|
||||
|
||||
Using Scrapy
|
||||
============
|
||||
|
||||
This section describes all key concepts of Scrapy.
|
||||
|
||||
Chapter 1. Main topics
|
||||
----------------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
architecture
|
||||
shell
|
||||
spiders
|
||||
items
|
||||
loaders
|
||||
selectors
|
||||
request-response
|
||||
|
||||
Chapter 2. Extension Mechanisms
|
||||
-------------------------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
item-pipeline
|
||||
downloader-middleware
|
||||
spider-middleware
|
||||
scheduler-middleware
|
||||
extensions
|
||||
|
||||
Chapter 3. Services
|
||||
-------------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
settings
|
||||
logging
|
||||
signals
|
||||
stats
|
||||
email
|
||||
|
||||
Chapter 4. Debugging and monitoring
|
||||
-----------------------------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
webconsole
|
||||
telnetconsole
|
||||
leaks
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
Chapter 5. Reference
|
||||
--------------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
exceptions
|
||||
link-extractors
|
||||
|
||||
Appendix 1: Third party tools
|
||||
-----------------------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
firefox
|
||||
firebug
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
.. _topics-signals:
|
||||
|
||||
.. module:: scrapy.core.signals
|
||||
:synopsis: Signals definitions
|
||||
|
||||
=======
|
||||
Signals
|
||||
=======
|
||||
@ -26,6 +23,9 @@ For more information about working when see the documentation of
|
||||
Built-in signals reference
|
||||
==========================
|
||||
|
||||
.. module:: scrapy.core.signals
|
||||
:synopsis: Signals definitions
|
||||
|
||||
Here's a list of signals used in Scrapy and their meaning, in alphabetical
|
||||
order.
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user