From 25f72148305975d785df8691a67afa8334585aa4 Mon Sep 17 00:00:00 2001 From: Sylvain Pace Date: Thu, 28 Mar 2019 14:32:31 +0100 Subject: [PATCH] Docs/update (#641) * fixing typo * keeping same convention * avoid wrong use of word crawling * removing superfluous comment * make the documentation more explicit to prevent error in the key usage, introduces feedback from algolia/docsearch-scraper#438, give instruction about how to use the headless chrome * add synonym, resolves #640 * precise sitemap * fix typos * removing unavailable anchor * fixing wrong URL * resolves #518 by removing unwanted char from anchors * fixing linter issues * fix remark-lint warning --- docs/scripts/lib/markdown.js | 6 +- docs/src/apply.md | 2 +- docs/src/behavior.md | 4 +- docs/src/config-file.md | 60 +++++++++--- docs/src/crawler-overview.md | 2 +- docs/src/faq.md | 27 +++--- docs/src/how-do-we-build-an-index.md | 6 -- docs/src/how-does-it-work.md | 4 +- docs/src/integrations.md | 10 +- docs/src/required-configuration.md | 11 +-- docs/src/run-your-own.md | 132 ++++++++++++++++----------- 11 files changed, 159 insertions(+), 105 deletions(-) diff --git a/docs/scripts/lib/markdown.js b/docs/scripts/lib/markdown.js index a8ea6b0a..afeaf71f 100644 --- a/docs/scripts/lib/markdown.js +++ b/docs/scripts/lib/markdown.js @@ -14,6 +14,11 @@ const markdown = markdownIt({ linkify: true, }) .use(markdownItAnchor, { + slugify: e => + e + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^\w-]/g, ''), permalink: true, permalinkClass: 'anchor', permalinkSymbol: '', @@ -28,7 +33,6 @@ export default { const headings = $('h2'); return _.map(headings, heading => { const $el = $(heading); - return { title: $el.text(), anchor: $el.attr('id'), diff --git a/docs/src/apply.md b/docs/src/apply.md index 34f93b47..958a5909 100644 --- a/docs/src/apply.md +++ b/docs/src/apply.md @@ -6,4 +6,4 @@ includeForm: true **Please make sure you [read the checklist][1] before applying.** -[1]: who-can-apply.html +[1]: ./who-can-apply.html diff --git a/docs/src/behavior.md b/docs/src/behavior.md index 5d947114..2ee46fb8 100644 --- a/docs/src/behavior.md +++ b/docs/src/behavior.md @@ -108,7 +108,7 @@ property of the `docsearch` instance. const search = docsearch({ […] autocompleteOptions: { - // See https://github.com/algolia/autocomplete.js#options + // See https://github.com/algolia/autocomplete.js#global-options } }); @@ -136,6 +136,6 @@ docsearch({ [1]: https://github.com/algolia/autocomplete.js [2]: ./run-your-own.html -[3]: https://github.com/algolia/autocomplete.js#options +[3]: https://github.com/algolia/autocomplete.js#global-options [4]: https://www.algolia.com/doc/api-reference/api-parameters/ [5]: https://www.algolia.com/doc/api-reference/api-parameters/hitsPerPage/ diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 8a85248d..69378445 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -3,9 +3,9 @@ layout: two-columns title: Config Files --- -For each DocSearch request we receive, we create a custom JSON config file that -will define how the crawler should behave. You can find all the configs in [this -repository][1]. +For each DocSearch request we receive, we create a custom JSON configuration +file that will define how the crawler should behave. You can find all the +configs in [this repository][1]. A DocSearch looks like this: @@ -121,9 +121,9 @@ be used. Do not forget to set this fallback set of selectors. ### Using regular expressions -The `start_urls` and `stop_urls` options also enable you to provide regular -expression to express more complex patterns. This object must at least contain a -`url` key targeting a reachable page. +The `start_urls` and `stop_urls` options also enable you to use regular +expressions to express more complex patterns. This object must at least contain +a `url` key targeting a reachable page. You can also define `variables` key that will be injected into your specific URL pattern. The following example makes this variable feature clearer: @@ -145,7 +145,7 @@ pattern. The following example makes this variable feature clearer: The beneficial side effect of using this syntax is that every records extracted from pages matching `http://www.example.com/docs/en/latest` will have attributes `lang: en` and `version: latest`. It enables you to filter on [these -`facetFilters`][3]. +`facetFilters`][4]. The following example shows how you can filter results matching specifics language and version from the frontend @@ -364,8 +364,8 @@ value by setting the `strip_chars` key. } ``` -Note that you can also define `strip_chars` directly at the root of the config -and it will be applied to all selectors. +Note that you can also define `strip_chars` directly at the root of the +configuration and it will be applied to all selectors. ```json { @@ -376,7 +376,7 @@ and it will be applied to all selectors. ### Targeting elements using XPath instead of CSS CSS selectors are a clear and concise way to target elements of a page, but they -have a limitations. For example, you cannot go _up_ the cascade with CSS. +have a limitations. For example, you cannot go up the cascade with CSS. If you need a more powerful selector mechanism, you can write your selectors using XPath by setting `type: xpath`. @@ -423,6 +423,31 @@ meaning. Check the [Algolia documentation][2] for more information on the settings. +`custom_settings` can include synonyms key that is an array of synonyms (up to +20 elements). Each element is an array of one word synonyms which can be +replaced by the others. + +For example: + +```json +"custom_settings": { + "synonyms": [ + [ + "js", + "javascript" + ], + [ + "es6", + "ECMAScript6", + "ECMAScript 6" + ] + ] + }, +``` + +_Note that you can use [advanced synonym thanks to Algolia][3]. Our scraper +supports only supports regular one word synonyms._ + ### `min_indexed_level` _Optional_ The default value is `0`. By increasing it, you can chose to not index some @@ -452,7 +477,7 @@ avoid being prompted, set the `UPDATE_NB_HITS` environment variable to `true` (to enable) or `false` (to disable). This variable can be set in the .env file alongside `APPLICATION_ID` and `API_KEY`. -You don't have to modify this field. We're documenting it here in case you were +You don't have to edit this field. We're documenting it here in case you were wondering what it's all about. ### `only_content_level` _Optional_ @@ -528,6 +553,9 @@ following every link of your `starts_urls`. } ``` +You must explicitly defined this parameter, our scraper doesn't follow +`robots.txt` + ### `sitemap_alternate_links` _Optional_ Sitemaps can contain _alternative links_ for URLs. Those are other versions of @@ -543,7 +571,7 @@ Set this to `true` if you want those other versions to be crawled as well. } ``` -With the above config and the `sitemap.xml` below, both +With the above configuration and the `sitemap.xml` below, both `http://www.example.com/docs/` and `http://www.example.com/docs/de/` will be crawled. @@ -561,8 +589,8 @@ that HTML source is returned directly by the server. If your content is generated by the front-end, you have to tell DocSearch to emulate a browser through Selenium. -_As client-side crawling is way slower than server-side crawling, we highly -encourage you to update your website to enable server-side rendering._ +_As client-side crawl is way slower than server-side crawl, we highly encourage +you to update your website to enable server-side rendering._ ### `js_render` _Optional_ @@ -631,4 +659,6 @@ To override it, from the configuration: [1]: https://github.com/algolia/docsearch-configs/tree/master/configs [2]: https://www.algolia.com/doc/api-reference/settings-api-parameters/ -[3]: https://www.algolia.com/doc/api-reference/api-parameters/facetFilters/ +[3]: + https://www.algolia.com/doc/guides/managing-results/optimize-search-results/adding-synonyms/#the-different-types-of-synonyms +[4]: https://www.algolia.com/doc/api-reference/api-parameters/facetFilters/ diff --git a/docs/src/crawler-overview.md b/docs/src/crawler-overview.md index 114758d1..05d1150e 100644 --- a/docs/src/crawler-overview.md +++ b/docs/src/crawler-overview.md @@ -16,7 +16,7 @@ will be used for the hierarchy, and each `p` of text will be used as a potential result. Those CSS selectors can be overwritten, and each website actually has its own -JSON configuration file that describes in more detail how the crawling should +JSON configuration file that describes in more detail how the crawl should behave. You can find the complete list of options in [the related section][2]. ## When? diff --git a/docs/src/faq.md b/docs/src/faq.md index 7162ba6c..4db78e38 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -12,8 +12,8 @@ us][1] directly. Every day. The exact time of day might vary each day, but we'll crawl your website at most -every 24 hours. We will also trigger a manual crawling every time your config is -updated. +every 24 hours. We will also trigger a manual crawl every time your +configurations is updated. ## What do I need to install on my side? @@ -36,7 +36,7 @@ displayed next to the search results. If this is not possible for you, you're free to [open your own Algolia account][3] and run DocSearch on your own without this limitation. In that case, though, depending on the size of your documentation, you might need a paid -account ([free accounts][4] can hold as much as 10k records). +account (free accounts can hold as much as 10k records). ## What data are you collecting? @@ -57,8 +57,8 @@ _If you don't have Analytics access, [send us an email][1] and we'll enable it._ ## Where is my data hosted? All DocSearch data is hosted on Algolia's servers, with replications around the -globe. You can find more details about the actual [server specs here][5], and -more complete information in our [privacy policy][6]. +globe. You can find more details about the actual [server specs here][4], and +more complete information in our [privacy policy][5]. ## Can I use DocSearch on non-doc pages? @@ -66,8 +66,8 @@ The free DocSearch we provide will crawl documentation pages. To use it on other parts of your website, you'll need to create your own Algolia account and either: -- Run the [DocSearch crawler][7] on your own -- Use one of our other [framework integrations or API clients][8] +- Run the [DocSearch crawler][6] on your own +- Use one of our other [framework integrations or API clients][7] ## Can you index code samples? @@ -107,7 +107,7 @@ is successful. These settings will be overridden at the next crawl. We **do not recommend to edit anything from the dashboard**. These changes have be made from the JSON configuration itself. -You can use the [custom_settings parameter][9] in such purpose. +You can use the [custom_settings parameter][8] in such purpose. ## A documentation website I like does not use DocSearch. What can I do? @@ -121,9 +121,8 @@ the help we can. [1]: mailto:docsearch@algolia.com [2]: https://www.algolia.com/ [3]: https://www.algolia.com/pricing -[4]: https://www.algolia.com/pricing#community -[5]: https://www.algolia.com/doc/guides/infrastructure/servers/ -[6]: https://www.algolia.com/policies/privacy -[7]: ./run-your-own.html -[8]: https://www.algolia.com/doc/api-reference/ -[9]: ./config-file.html#custom_settings-optional +[4]: https://www.algolia.com/doc/guides/infrastructure/servers/ +[5]: https://www.algolia.com/policies/privacy +[6]: ./run-your-own.html +[7]: https://www.algolia.com/doc/api-reference/ +[8]: ./config-file.html#custom_settings-optional diff --git a/docs/src/how-do-we-build-an-index.md b/docs/src/how-do-we-build-an-index.md index 81ea7783..60a88e00 100644 --- a/docs/src/how-do-we-build-an-index.md +++ b/docs/src/how-do-we-build-an-index.md @@ -9,12 +9,6 @@ In this section you will learn how we build a DocSearch index from your page. ![1st step][2] {mt-2} -_Note: We would rather avoid useless JavaScript rendering. All useful -information should be available without any client-side rendering. You can use -`curl` command to have a better idea of how it would look without any -client-side rendering. You can also disable JavaScript from your browser -settings_ - ## We extract the payload thanks to your set of `selectors` ![2nd step][3] {mt-2} diff --git a/docs/src/how-does-it-work.md b/docs/src/how-does-it-work.md index 23d8e8aa..af4ca0da 100644 --- a/docs/src/how-does-it-work.md +++ b/docs/src/how-does-it-work.md @@ -48,7 +48,7 @@ crawling your website every day and update your search results automatically. All we ask is that you keep the "search by Algolia" logo next to your search results. -[1]: apply.html +[1]: ./apply.html [2]: https://github.com/algolia/docsearch-configs/tree/master/configs -[3]: styling.html +[3]: ./styling.html [4]: ./assets/docsearch-how-it-works.png diff --git a/docs/src/integrations.md b/docs/src/integrations.md index 60a16b4c..cb4bb787 100644 --- a/docs/src/integrations.md +++ b/docs/src/integrations.md @@ -6,7 +6,7 @@ title: Supported Integrations We worked with **documentation website generators** to have DocSearch directly embedded as a first class citizen in the websites they produce. -## Our great integrations: +## Our great integrations So, if you're using one of the following tools, checkout their documentation to see how to enable DocSearch on your website: @@ -25,8 +25,8 @@ to the list, get [in touch with us][10]. We'd be happy to help. [3]: https://vuepress.vuejs.org/ [4]: https://vuepress.vuejs.org/default-theme-config/#search-box [5]: https://docs.gitbook.com/ -[6]: http://pkgdown.r-lib.org/index.html -[7]: http://pkgdown.r-lib.org/articles/pkgdown.html#search -[8]: https://larecipe.binarytorch.com.my/docs/1.2/overview -[9]: https://larecipe.binarytorch.com.my/docs/1.2/configurations#search +[6]: https://pkgdown.r-lib.org/ +[7]: https://pkgdown.r-lib.org/articles/pkgdown.html?q=search#docsearch-indexing +[8]: https://larecipe.binarytorch.com.my/docs/1.3/overview +[9]: https://larecipe.binarytorch.com.my/docs/1.3/configurations#search [10]: mailto:docsearch@algolia.com diff --git a/docs/src/required-configuration.md b/docs/src/required-configuration.md index d3ad12e6..c8b7d7bf 100644 --- a/docs/src/required-configuration.md +++ b/docs/src/required-configuration.md @@ -9,7 +9,7 @@ following specification is required to let our crawler build the best experience from your website. You will need to update your website and follow these rules. Your documentation will match our generic configuration: -Note: If your website is generated thank to one of our supported tool, you do +Note: If your website is generated thanks to one of our supported tool, you do not need to change your website as it is already compliant with our requirements. @@ -58,10 +58,9 @@ in the following guidelines. ### Implement the right classes as [selectors][3] -You can add some specific static classes to help us identify your content's -role. These classes can not involve any style changes. These dedicated classes -will help us to create a great learn as you type experience from your -documentation. +You can add some specific static classes to help us find your content's role. +These classes can not involve any style changes. These dedicated classes will +help us to create a great learn as you type experience from your documentation. - Add a static class `DocSearch-content` to the main container of your textual content. It is mostly a `
` or an `
` HTML element. @@ -124,7 +123,7 @@ exposed via the DocSearch `custom_settings` parameter. - Make sure your documentation content is also available without JavaScript rendering on the client-side. If you absolutely need JavaScript turned on, you - need to [set `js_render: true` in your config][6]. + need to [set `js_render: true` in your configuration][6]. Any question ? [Send us an email][7]. diff --git a/docs/src/run-your-own.md b/docs/src/run-your-own.md index 12033f9e..5a14899c 100644 --- a/docs/src/run-your-own.md +++ b/docs/src/run-your-own.md @@ -3,51 +3,74 @@ layout: two-columns title: Run your own --- -The version of DocSearch we provide for free is one hosted on our own servers, -running every 24 hours. To update your results with more control, or if you are -not compliant with our checklist, or to index content sitting behind a firewall, -you might want to run the crawler yourself. +The version of DocSearch we offer for free is hosted on our own servers, running +every 24 hours. You do not need to think about the crawl. However if you want to +update your results with more control, or if you are [not compliant with our +checklist][1], or if your website sits behind a firewall, you might want to run +the crawler yourself. -The whole code of DocSearch is open source, and we packaged it as a Docker image -to make this even easier for you to use. +The whole code base of DocSearch is open source, and we package it as a Docker +image to make this even easier for you to use. -## Installation - -The scraper is a python tool [based on scrapy][1]. Start by cloning [the open -source repository][2]. We do recommend [pipenv][3] to install the whole python -environment - -- [Install pipenv][4] -- `pipenv install` -- `pipenv shell` - -You should be ready to go. - -You can use DocSearch from inside a Docker image. You can setup one by running -`./docsearch docker:build`. - -## Configuration +## Set up your environment You'll need to set your Algolia application ID and admin API key as environment -variables. If you don't have an Algolia account, you should [create one][5]. +variables. If you don't have an Algolia account, you need to [create one][2]. -- `APPLICATION_ID` should be set to your Application ID +- `APPLICATION_ID` set to your Algolia Application ID -- `API_KEY` should be set to your API Key. Make sure to use an API key with - **write** access to your index. +- `API_KEY` set to your API Key. Make sure to use an API key with **write** + access to your index. [The ACL `addObject`, `editSettings` and + `deleteIndex`][3] must be allowed to this key. -For convenience, you can create a `.env` file in the repository root with the -following format and DocSearch will use those values. +For convenience, you can create a `.env` file in the repository root. ```sh APPLICATION_ID=YOUR_APP_ID API_KEY=YOUR_API_KEY ``` -## Create a new config +## Run the crawl from the Docker image -To create a config, run `./docsearch bootstrap`. A prompt will ask you for some -information and will then output a JSON config you can use as a base. +You can run a crawl from the packaged Docker image to crawl your website. You +will need to [install jq, a lightweight command-line JSON processor][4] + +Then you only need to start the crawl according to your configuration. You +should check the [dedicated configuration documentation][5]. + +```sh +docker run -it --env-file=.env -e "CONFIG=$(cat /path/to/your/config.json | jq -r tostring)" algolia/docsearch-scraper +``` + +## Installation + +The scraper is a python tool [based on scrapy][6]. We do recommend to use +[pipenv][8] to install the python environment. + +- [Clone the scraper repository][7]. +- [Install pipenv][9] +- `pipenv install` +- `pipenv shell` + +If you plan to use the browser emulation [(`js_render` set to true)][10], you +need to follow this extra step. If you don't, you can dismiss this step. + +### Installing Chrome driver + +Some websites rendering requires JavaScript. Our crawler rely on a headless +chrome emulation. You will need to set up a ChromeDriver. + +- [Install the driver][11] suited to your OS and the version of your Chrome. We + do recommend to use the latest version. +- Set the environment variable `CHROMEDRIVER_PATH` in your `.env` file. This + path must target the downloaded extracted driver. + +You are ready to go. + +## Create a new configuration + +To create a configuration, run `./docsearch bootstrap`. A prompt will ask you +for some information and will create a JSON configuration you can use as a base. ```sh $ ./docsearch bootstrap @@ -69,6 +92,7 @@ index_name is example [enter to confirm]: "lvl2": "FIXME h3", "lvl3": "FIXME h4", "lvl4": "FIXME h5", + "lvl5": "FIXME h6", "text": "FIXME p, FIXME li" } } @@ -76,13 +100,12 @@ index_name is example [enter to confirm]: ``` Create a file from this text into a filename `example.json`, we'll use it later -on to start the crawl. You can find the complete list of available options in -[the configuration documentation][6], or browse the [list of live configs][7]. +on to start the crawl. You can browse the [list of live configurations][12]. -## Running your config +## Run the crawl from the code base Now that you have your environment variables set, you can run the crawler -according to your config. +according to your configuration. Running `pipenv shell` will enable your virtual environment. From there, you can run one crawl with the following command: @@ -91,7 +114,7 @@ run one crawl with the following command: $ ./docsearch run /path/to/your/config.json ``` -Or from your built docker image: +Or from the Docker image: ```sh $ ./docsearch docker:run /path/to/your/config.json @@ -105,7 +128,7 @@ built records to Algolia. You can test your results by running `./docsearch playground`. This will open a web page with a search input. You can do live tests against the indexed results. -![Playground][9] {mt-2} +![Playground][13] {mt-2} _Note that if the command fails (it can happen on non-Mac machines), you can get the same result by running a live server in the `./playground` subdirectory.\`_ @@ -113,17 +136,17 @@ the same result by running a live server in the `./playground` subdirectory.\`_ ## Integration Once you're satisfied with your config, you can integrate the dropdown menu in -your website by following the [instructions here][8]. +your website by following the [instructions here][14]. The difference is that you'll also have to add the `appId` key to your `docsearch()` instance. Also don't forget to use a **search** API key here (in -other words, not the **write** API key you used for the crawling). +other words, not the **write** API key you used for the crawl). ```javascript docsearch({ appId: '', // Add your own Application ID - apiKey: '', // Set it to your own search API key - […] // Other settings are identical + apiKey: '', // Set it to your own *search* API key + […] // Other parameters are the same }); ``` @@ -132,15 +155,20 @@ docsearch({ You can run `./docsearch` without any argument to see the list of all available commands. -Note that we use this command-line tool internally at Algolia to run the free -hosted version, so you might not need all the listed commands. +_Note that we use this command-line tool internally at Algolia to run the free +hosted version, so you might not need all the listed commands._ -[1]: https://scrapy.org/ -[2]: https://github.com/algolia/docsearch-scraper -[3]: https://github.com/pypa/pipenv -[4]: https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv -[5]: https://www.algolia.com/pricing#community -[6]: ./config-file.html -[7]: https://github.com/algolia/docsearch-configs/tree/master/configs -[8]: ./dropdown.html -[9]: ./assets/playground.png +[1]: ./who-can-apply.html +[2]: https://www.algolia.com/pricing +[3]: https://www.algolia.com/doc/guides/security/api-keys/#acl +[4]: https://github.com/stedolan/jq/wiki/Installation +[5]: ./config-file.html +[6]: https://scrapy.org/ +[7]: https://github.com/algolia/docsearch-scraper +[8]: https://github.com/pypa/pipenv +[9]: https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv +[10]: ./config-file.html#js_render-optional +[11]: http://chromedriver.chromium.org/getting-started +[12]: https://github.com/algolia/docsearch-configs/tree/master/configs +[13]: ./assets/playground.png +[14]: ./dropdown.html