From 4e0d9bb30bf160c583b4081c314f4f76917e7c89 Mon Sep 17 00:00:00 2001
From: Sylvain Pace <sylvain.pace@algolia.com>
Date: Wed, 18 Jul 2018 10:47:59 +0200
Subject: [PATCH] documenting algolia/docsearch-scraper#387

---
 .../2-config-options.html.md.erb              | 106 +-----------------
 1 file changed, 4 insertions(+), 102 deletions(-)

diff --git a/docs/source/documentation/2-docsearch-scraper/2-config-options.html.md.erb b/docs/source/documentation/2-docsearch-scraper/2-config-options.html.md.erb
index 8d33dcd8..584ae71c 100644
--- a/docs/source/documentation/2-docsearch-scraper/2-config-options.html.md.erb
+++ b/docs/source/documentation/2-docsearch-scraper/2-config-options.html.md.erb
@@ -99,108 +99,6 @@ The whole pattern `(?P<version>.*?)` will be replaced by the value assigned in t
 The variable name is not fixed. Please note that those variables will behave as
 [`attributesForFaceting`](https://www.algolia.com/doc/api-reference/api-parameters/attributesForFaceting/) which may help you [restrain the scope of the search from the snippet](https://www.algolia.com/doc/guides/searching/faceting/).
 
-Thus you can limit the scope of the search to the records from the pages encompassed by `http://example.com/docs/en/latest/*` thanks to the following snippet:
-```js
-<script type="text/javascript">
- docsearch({
-    apiKey: ${apiKey},
-    indexName: ${indexName},
-    inputSelector: '#search'
-    algoliaOptions: { 'facetFilters': ["lang:en", "version:latest"] },
-});
-</script>
- ```
-
-In order to promote some pages, you can set the `page_rank` attribute (default: `0`, can be a positive or negative integer).
-
-Finally, If your website contains differents parts and layouts, you can define specific `selectors` for each part and apply them using `selectors_key`:
-
-**Example:**
-```json
-{
-  "index_name": "example",
-   "start_urls": [
-    "http://example.com/docs/latest/",
-    {
-     "url": "http://example.com/docs/concepts/",
-     "page_rank": 1,
-     "selectors_key": "concepts"
-    }
-   ],
-   "selectors": {
-     "default": {
-       "lvl0": ".docSearch-content h1",
-       "lvl1": ".docSearch-content h2",
-       "lvl2": ".docSearch-content h3",
-       "lvl3": ".docSearch-content h4",
-       "lvl4": ".docSearch-content h5",
-       "text": ".docSearch-content p, .docSearch-content li"
-      },
-     "concepts": {
-       "lvl0": ".docSearch-header h2",
-       "lvl1": ".docSearch-content h1",
-       "lvl2": ".docSearch-content h2",
-       "lvl3": ".docSearch-content h3",
-       "lvl4": ".docSearch-content h5",
-       "text": ".docSearch-content p"
-     }
-   }
-  [...]
-}
-```
-
-#### `start_urls.tags`
-
-Tags will be apllied to every record from the matchin matched pages, i.e., its URL is matchin the `url`. These `tags` will [be processed as `attributesForFaceting`](https://www.algolia.com/doc/api-reference/api-parameters/attributesForFaceting/) . If you want further detail, [check the original documentation](https://www.algolia.com/doc/api-reference/api-parameters/facetFilters/).
-We do recommend to use `tags` if you want [to refine the scope of your search](https://www.algolia.com/doc/guides/searching/faceting/#faceting-overview).
-
-From your search UI, you will need to use the following input:
-```js
-algoliaOptions: { 'facetFilters': ["tags:$VALUE"] },
-```
-
-Default not used. Must be an array.
-
-
-**Example:**
-
- From the config:
-```json
-"start_urls": [
- {
-  "url": "http://example.com/docs/concepts/",
-  "selectors_key": "concepts"
- }
- ```
-
- From the search UI:
- ```js
- algoliaOptions: { 'facetFilters': ["tags:concepts"] },
- ```
-### `scrape_start_urls`
-
-**Example:**
-```json
-{
-   "url": "http://example.com/docs/(?P<lang>.*?)/(?P<version>.*?)/",
-   "variables": {
-      "version": [
-         "latest",
-         "3.3",
-         [...]
-      ],
-      "lang": [
-         "en",
-         "fr",
-         [...]
-      ]
-}
-```
-The whole pattern `(?P<version>.*?)` will be replaced by the value assigned in the related key `version`.
-
-The variable name is not fixed. Please note that those variables will behave as
-[`attributesForFaceting`](https://www.algolia.com/doc/api-reference/api-parameters/attributesForFaceting/) which may help you [restrain the scope of the search from the snippet](https://www.algolia.com/doc/guides/searching/faceting/).
-
 Thus you can limit the scope of the search to the records from the pages encompassed by `http://example.com/docs/en/latest/*` thanks to the following snippet:
 ```js
 <script type="text/javascript">
@@ -411,6 +309,10 @@ Default is `false`
 
 ### `allowed_domains` _Optional_
 
+You can pass an array of strings. This is the whitelist of
+domains the crawler will browse. If a link targets a page that is not in the
+whitelist, the crawler will not follow it.
+
 ### Sitemap crawling _Optional_
 
 ### Sitemap crawling _Optional_