fixes referencing, and list only settings not documented in current document

docs/_ext/scrapydocs.py

@@ -13,8 +13,24 @@ class SettingsListDirective(Directive):
         return [settingslist_node('')]
 
 
-def is_setting_node(node):
-    return node.tagname == 'pending_xref' and node['reftype'] == 'setting'
+def is_setting_index(node):
+    if node.tagname == 'index':
+        # index entries for setting directives look like:
+        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
+        entry_type, info, refid, _ = node['entries'][0]
+        return entry_type == 'pair' and info.endswith('; setting')
+    return False
+
+
+def get_setting_target(node):
+    # target nodes are placed next to the node in the doc tree
+    return node.parent[node.parent.index(node) + 1]
+
+
+def get_setting_name_and_refid(node):
+    """Extract setting name from directive index node"""
+    entry_type, info, refid, _ = node['entries'][0]
+    return info.replace('; setting', ''), refid
 
 
 def collect_scrapy_settings_refs(app, doctree):
@@ -23,29 +39,24 @@ def collect_scrapy_settings_refs(app, doctree):
     if not hasattr(env, 'scrapy_all_settings'):
         env.scrapy_all_settings = []
 
-    for node in doctree.traverse(is_setting_node):
-        try:
-            targetnode = node.parent[node.parent.index(node) - 1]
-            if not isinstance(targetnode, nodes.target):
-                raise IndexError
-        except IndexError:
-            targetid = "setting-%d" % env.new_serialno('setting')
-            targetnode = nodes.target('', '', ids=[targetid])
-            node.replace_self([targetnode, node])
+    for node in doctree.traverse(is_setting_index):
+        targetnode = get_setting_target(node)
+        assert isinstance(targetnode, nodes.target), "Next node is not a target"
+
+        setting_name, refid = get_setting_name_and_refid(node)
 
         env.scrapy_all_settings.append({
             'docname': env.docname,
-            'lineno': node.line,
-            'node': node.deepcopy(),
-            'target': targetnode,
+            'setting_name': setting_name,
+            'refid': refid,
         })
 
 
 def make_setting_element(setting_data, app, fromdocname):
-    text = nodes.Text(setting_data['node'].astext())
-    targetid = ''  # TODO: resolve to a proper id
     refnode = make_refnode(app.builder, fromdocname,
-                           setting_data['docname'], targetid, text)
+                           todocname=setting_data['docname'],
+                           targetid=setting_data['refid'],
+                           child=nodes.Text(setting_data['setting_name']))
 
     p = nodes.paragraph()
     p.append(refnode)
@@ -57,7 +68,8 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
 
     for node in doctree.traverse(settingslist_node):
         node.replace_self([make_setting_element(d, app, fromdocname)
-                           for d in env.scrapy_all_settings])
+                           for d in env.scrapy_all_settings if fromdocname != d['docname']])
+
 
 def setup(app):
     app.add_crossref_type(

docs/topics/settings.rst

@@ -133,8 +133,6 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
-.. settingslist::
-
 .. setting:: AWS_ACCESS_KEY_ID
 
 AWS_ACCESS_KEY_ID
@@ -1017,6 +1015,16 @@ Default: ``"Scrapy/VERSION (+http://scrapy.org)"``
 
 The default User-Agent to use when crawling, unless overridden.
 
+
+Settings documented elsewhere:
+------------------------------
+
+The following settings are documented elsewhere, please check each specific
+case to see how to enable and use them.
+
+.. settingslist::
+
+
 .. _Amazon web services: http://aws.amazon.com/
 .. _breadth-first order: http://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: http://en.wikipedia.org/wiki/Depth-first_search