diff --git a/docs/config.json b/docs/config.json index 6d66cc6c..ca483b9f 100644 --- a/docs/config.json +++ b/docs/config.json @@ -2,10 +2,10 @@ "site": { "title": "DocSearch", "description": "The easiest way to add search to your documentation", - "url": "https://community.algolia.com/docsearch" - }, - "placeholders": { + "url": "https://community.algolia.com/docsearch", + "repo": "https://github.com/algolia/docsearch" }, + "placeholders": {}, "sidebar": [ { "title": "Essentials", @@ -15,17 +15,17 @@ "url": "what-is-docsearch.html" }, { - "title": "How it works", - "url": "how-it-works.html" + "title": "How does it work?", + "url": "how-does-it-work.html" + }, + { + "title": "Apply now", + "url": "apply.html" }, { "title": "Tips", "url": "tips.html" }, - { - "title": "Who qualifies", - "url": "qualification.html" - }, { "title": "FAQ", "url": "faq.html" @@ -33,10 +33,6 @@ { "title": "FAQ2", "url": "faq2.html" - }, - { - "title": "Config help", - "url": "configuration-help.html" } ] }, @@ -62,11 +58,15 @@ "pages": [ { "title": "Overview", - "url": "crawler.md" + "url": "crawler-overview.html" + }, + { + "title": "Inside the engine", + "url": "inside-the-engine.html" }, { "title": "Config files", - "url": "crawler-config.html" + "url": "config-file.html" }, { "title": "Docker image", diff --git a/docs/scripts/lib/markdown.js b/docs/scripts/lib/markdown.js index 42dfcff8..51eb381d 100644 --- a/docs/scripts/lib/markdown.js +++ b/docs/scripts/lib/markdown.js @@ -8,7 +8,10 @@ import markdownItHighlight from 'markdown-it-highlightjs'; import cheerio from 'cheerio'; import pug from 'pug'; import pMap from 'p-map'; -const markdown = markdownIt() +const markdown = markdownIt({ + html: true, + linkify: true, +}) .use(markdownItAnchor, { permalink: true, permalinkClass: 'anchor', diff --git a/docs/src/_includes/header.pug b/docs/src/_includes/header.pug index a2298cb5..b992e9e7 100644 --- a/docs/src/_includes/header.pug +++ b/docs/src/_includes/header.pug @@ -16,8 +16,8 @@ .flex.flrw.pt-0x a.fln.w-50.flex.flrnw.p-05.flc.hover_no-underline(href="https://community.algolia.com/talksearch/") .fln.bg-teal.rounded-auto.w-2x.h-2x - img.w-100(src="assets/docsearch-icon.svg", alt="talksearch") - .fla.pl-1.text-solstice.text-2.bold.hover_text-nebula Talksearch + img.w-100(src="assets/talksearch-icon.svg", alt="TalkSearch") + .fla.pl-1.text-solstice.text-2.bold.hover_text-nebula TalkSearch a.fln.w-50.flex.flrnw.p-05.flc.hover_no-underline(href="https://community.algolia.com/jekyll-algolia/") .fln.bg-purple.rounded-auto.w-2x.h-2x img.w-100(src="assets/jekyll-icon.svg", alt="Jekyll") @@ -45,9 +45,9 @@ .hidden.lg_flex.fln.flrnw.flc .w-1.h-1.sm_w-1x.sm_h-1x.text-telluric.fill-current include ../assets/arrow-separator.svg - //- TalkSearch link + //- Project link .flex.fln.flrnw.flc.pl-1.sm_pl-0 - a.cursor-pointer.uppercase.tracking-poppins.poppins.text-solstice.hover_text-nebula.hover_no-underline(href=site.url) TalkSearch + a.cursor-pointer.uppercase.tracking-poppins.poppins.text-solstice.hover_text-nebula.hover_no-underline(href=site.url)=site.title //- Filler .flex.fla.md_hidden @@ -61,7 +61,7 @@ //- Links .flex.fln.flrnw.flc.px-2 - a.cursor-pointer.text-2.text-solstice.hover_text-nebula.hover_no-underline(href="./what-is-talksearch.html") Documentation + a.cursor-pointer.text-2.text-solstice.hover_text-nebula.hover_no-underline(href="./what-is-docsearch.html") Documentation .flex.fln.flrnw.flc.pr-0x - a.w-1x.h-1x.text-solstice.fill-current(href="https://github.com/algolia/talksearch", target="_blank") + a.w-1x.h-1x.text-solstice.fill-current(href=site.repo, target="_blank") include ../assets/github-logo.svg diff --git a/docs/src/_layouts/two-columns.pug b/docs/src/_layouts/two-columns.pug index 5ec898a7..bb379a01 100644 --- a/docs/src/_layouts/two-columns.pug +++ b/docs/src/_layouts/two-columns.pug @@ -1,26 +1,27 @@ extends main.pug block content - .bgtb-white-moon.pt-3x.pb-3.poppins + .bgtb-white-pure-moon.pt-3x.pb-3.poppins .max-w-lg.m-auto.text-center .text-6.lg_text-8.uppercase.tracking-poppins.text-solstice.bold=current.title - .flex.flrnw.max-w-xl.m-auto - .fln.w-30 - // Adding subsections from markdown - .sticky.pin-t.top-3.h-100vh-3.pl-1.lg_pl-2.overflow-y-auto - each category in sidebar - .category.text-nebula.poppins.uppercase.mt-1=category.title - ul.list-reset.pt-05.pb-1.text-2 - each page in category.pages - - var isCurrent = current.url === page.url - - var additionalClasses = isCurrent ? 'text-nebula' : 'text-solstice' - li - a.block.py-0x(href=page.url, class=additionalClasses)=page.title - - var hasSubLinks = (page.headings && page.headings.length) - if hasSubLinks - ul.list-reset.pl-1.text-2 - each heading in page.headings - li - a.block.py-0x.text-telluric(href=`${page.url}#${heading.anchor}`)=heading.title + .bgtb-moon-white-pure + .flex.flrnw.max-w-lg.m-auto + .fla + // Adding subsections from markdown + .sticky.pin-t.top-3.h-100vh-3.pl-1.lg_pl-2.overflow-y-auto + each category in sidebar + .category.text-nebula.poppins.uppercase.mt-1=category.title + ul.list-reset.pt-05.pb-1.text-2 + each page in category.pages + - var isCurrent = current.url === page.url + - var additionalClasses = isCurrent ? 'text-nebula' : 'text-solstice' + li + a.block.py-0x(href=page.url, class=additionalClasses)=page.title + - var hasSubLinks = (page.headings && page.headings.length) + if hasSubLinks + ul.list-reset.pl-1.text-2 + each heading in page.headings + li + a.block.py-0x.text-telluric(href=`${page.url}#${heading.anchor}`)=heading.title - .documentation-content.fla.text-2.pb-2.pr-1.sm_px-2.md_px-3.overflow-hidden - !=current.content + .documentation-content.fln.w-md.text-2.pb-2.pr-1.sm_px-2.overflow-hidden + !=current.content diff --git a/docs/src/qualification.md b/docs/src/apply.md similarity index 98% rename from docs/src/qualification.md rename to docs/src/apply.md index aab350ae..7885308b 100644 --- a/docs/src/qualification.md +++ b/docs/src/apply.md @@ -1,8 +1,10 @@ --- layout: two-columns -title: Does my project qualify? +title: Applying to DocSearch --- +// TODO + ## Our policy We really want to improve documentation with search. Finding the information you are looking for is the very first step when reading docs. Search is key. diff --git a/docs/src/assets/angular-icon.svg b/docs/src/assets/angular-icon.svg new file mode 100644 index 00000000..dba47603 --- /dev/null +++ b/docs/src/assets/angular-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/docs/src/assets/instantsearch-icon.svg b/docs/src/assets/instantsearch-icon.svg new file mode 100644 index 00000000..ce30c4d0 --- /dev/null +++ b/docs/src/assets/instantsearch-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/docs/src/assets/jekyll-icon.svg b/docs/src/assets/jekyll-icon.svg new file mode 100644 index 00000000..10c602d2 --- /dev/null +++ b/docs/src/assets/jekyll-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/docs/src/assets/vue-icon.svg b/docs/src/assets/vue-icon.svg new file mode 100644 index 00000000..0d0b5ccf --- /dev/null +++ b/docs/src/assets/vue-icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/docs/src/config-file.md b/docs/src/config-file.md index e686c94f..c739943b 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -3,6 +3,8 @@ layout: two-columns title: Make the most of your configuration --- +// TODO + ## Introduction The DocSearch scraper will use a configuration file specifying: diff --git a/docs/src/crawler-overview.md b/docs/src/crawler-overview.md new file mode 100644 index 00000000..b06e708f --- /dev/null +++ b/docs/src/crawler-overview.md @@ -0,0 +1,32 @@ +--- +layout: two-columns +title: Crawler Overview +--- + +The DocSearch crawler is written in python and heavily based on the +[Scrapy][1] framework. It will crawl all pages of your +website, and extract content from the HTML structure to populate an Algolia +index. + +It will automatically follow every internal link to make sure we are not missing +any content, and will use the semantics of your HTML structure to construct its +records. This means that `h1`...`h6` titles will be used for the hierarchy, and +each `p` of text will be used as a potential result. + +Those CSS selectors can of course be overwritten, and each website actually has +its own JSON configuration file that describe in more details how the crawling +should behave. You can find the complete list of options in [our +documentation][2]. + +We automatically run each config every 24h. This is done from our own +infrastructure, meaning that you don't need to install anything on your side. +We run this service entirely free of charge, we're just asking that you keep the +"powered by Algolia" logo next to the search results. + +That being said, if you'd like to run DocSearch on your own, [all the code is +open-source][3] and even packaged as a Docker image. Just grab it, and run it +with your own credentials. + +[1]: https://scrapy.org/ +[2]: ./crawler-config.html +[3]: https://github.com/algolia/docsearch-scraper diff --git a/docs/src/docker.md b/docs/src/docker.md index 9de5d9d7..e70d1302 100644 --- a/docs/src/docker.md +++ b/docs/src/docker.md @@ -3,6 +3,8 @@ layout: two-columns title: DocSearch on your own infrastructure --- +// TODO + When we entirely host your DocSearch implementation, you don't have to bother, we **handle the whole stack**. In some specific use cases and situations, you may need to look under the hood and DIY. diff --git a/docs/src/events.md b/docs/src/events.md index 23bc7dff..1b6fe0e3 100644 --- a/docs/src/events.md +++ b/docs/src/events.md @@ -3,6 +3,8 @@ layout: two-columns title: Configuring the search results --- +// TODO + DocSearch is a wrapper around the [autocomplete.js](https://github.com/algolia/autocomplete.js) library that gets its results from the Algolia API. As such, you can use any options provided by this project and by [the Algolia API](https://www.algolia.com/doc/api-reference/). diff --git a/docs/src/faq.md b/docs/src/faq.md index e2ad4db8..7ab1af25 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -3,6 +3,8 @@ layout: two-columns title: Global wonderings --- +// TODO + ## What is the general behaviour? ### How often do you scrap my website? diff --git a/docs/src/how-does-it-work.md b/docs/src/how-does-it-work.md new file mode 100644 index 00000000..7bc80365 --- /dev/null +++ b/docs/src/how-does-it-work.md @@ -0,0 +1,55 @@ +--- +layout: two-columns +title: How does it work? +--- + +Getting up and ready with DocSearch is a straightforward process that requires +a few steps: you apply, we configure the crawler for you, and you update your +front-end. + +How it works + + +## 1. You apply to DocSearch + +The first thing you'll need to do is to apply for DocSearch by filling the +form on this page (make sure to double check that you qualify first). We are +receiving a lot of requests, so this form makes sure we won't be forgetting +anyone. + +We guarantee that we will answer to every request, but due to the amount of +applications we receive, please give us a couple of days to get back to you :) + +## 2. We create a configuration for you + +Once we receive your application, we'll have a look at your website and create +a custom configuration file for it. This file defines which urls we +should crawl or ignore, as well as the specific CSS selectors to be used for +selecting headers, subheaders, etc. All configs are publicly available in our +[config repo][1]. + +This step still requires some manual work, but thanks to the 900+ configs we +already created, we're able to automate most of it. Once done, we'll run a first +indexing of your website and have it run automatically every 24h. + +## 3. You add docsearch.js to your website + +We'll then get back to you with the JavaScript snippet you'll need to add to +your website. This will bind your search `input` field to display results from +your Algolia index on each keystroke in a dropdown menu. + +The default styling of the dropdown uses grey colors to fit in most designs. The +dropdown itself is made of HTML with custom CSS classes and we recommend that +you overwrite those classes to provide a theming more inline with the rest of +your website. + +## 4. We all enjoy great search + +Now that DocSearch is set, you don't have anything else to do. We'll keep +crawling your website every day and update your search results automatically. + + +[1]: https://github.com/algolia/docsearch-configs/tree/master/configs +[2]: https://github.com/algolia/docsearch-scraper +[3]: https://github.com/algolia/docsearch-scraper +[4]: https://www.sitemaps.org/ diff --git a/docs/src/how-it-works.md b/docs/src/how-it-works.md deleted file mode 100644 index d0dee0f2..00000000 --- a/docs/src/how-it-works.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -layout: two-columns -title: How does it work? ---- - -## How to start DocSearch? - -It all starts with a simple config file. The config file contains your -doc site's structure, for example your specific CSS selectors. It also -contains a number of attributes and directives that tell -[our DocSearch dedicated](https://github.com/algolia/docsearch-scraper) -tool how to find the searchable information from your website. - -## The Scraper, to extract content - -At a **regular time interval** (24h), our scraper will index the content of your -whole website. This **brand new** index will collect the public data of your -documentation website and push it onto our dedicated server in order to enable -your DocSearch instance. The process follows the directives contained in your -**customized configuration** file to tailor the generic behavior of our scraper, -with the aim of taking the most important information and organizing it for search optimization. - -This scraping process runs on our infrastructure, not yours. You don't have to bother -with it. However, our [scraper is also totally open source](https://github.com/algolia/docsearch-scraper) -and so you are free to run it on your own servers. - -## Browsing through your pages - -In order to discover your pages, we highly recommend the use of a [sitemap](https://www.sitemaps.org/) -which will be our source of truth and is a good practice for SEO purposes. - -In any case, our tool is crawling your website. It follows every hyperlink -embedded within the scraped pages as long as the URL belongs to the scope -defined by the `start_urls` (and/or `sitemap_urls_regexs` applied to your sitemap). - -## Building your index - -Building records using the scraper is pretty intuitive. According to your settings, -we extract the payload of your webpage and index it, preserving your data's structure. -This is achieved in a simple way: -* We **read top down** your web page following your HTML flow and pick out your -matching elements according their **levels** (`selectors_level`) -* We create a record for each paragraph along with its hierarchical path. -This construction is based on their **time of appearance** along the flow. -* We **index** these records with the appropriate global settings (e.g. metadata, tags, etc.) - -_**Note:** The above process performs sanity tests as it scrapes, in order to detect errors. -If indeed there are any serious warnings, it will abort and therefore not overwrite your current index. -These checks ensure that your dedicated index isn't flushed._ diff --git a/docs/src/inside-the-engine.md b/docs/src/inside-the-engine.md new file mode 100644 index 00000000..3e8d0c1f --- /dev/null +++ b/docs/src/inside-the-engine.md @@ -0,0 +1,21 @@ +--- +layout: two-columns +title: Inside the engine +--- + +// TODO + +## Building your index + +Building records using the scraper is pretty intuitive. According to your settings, +we extract the payload of your webpage and index it, preserving your data's structure. +This is achieved in a simple way: +* We **read top down** your web page following your HTML flow and pick out your +matching elements according their **levels** (`selectors_level`) +* We create a record for each paragraph along with its hierarchical path. +This construction is based on their **time of appearance** along the flow. +* We **index** these records with the appropriate global settings (e.g. metadata, tags, etc.) + +_**Note:** The above process performs sanity tests as it scrapes, in order to detect errors. +If indeed there are any serious warnings, it will abort and therefore not overwrite your current index. +These checks ensure that your dedicated index isn't flushed._ diff --git a/docs/src/leftovers.md b/docs/src/leftovers.md new file mode 100644 index 00000000..9b90c8f8 --- /dev/null +++ b/docs/src/leftovers.md @@ -0,0 +1,6 @@ +--- +layout: two-columns +--- + + + diff --git a/docs/src/style.css b/docs/src/style.css index 6185e9f9..a766bfbc 100644 --- a/docs/src/style.css +++ b/docs/src/style.css @@ -84,10 +84,11 @@ a:focus { @apply .underline; } code { - @apply .code .text--1 .text-mars-2; + @apply .code .text-mars-2; @apply .p-0x; @apply .bg-moon; @apply .rounded-1; + font-size: .9em; } /* Content converted form markdown */ @@ -102,12 +103,14 @@ code { /* Titles */ h2 { @apply .text-solstice .text-5 .normal; - @apply .mb-1; + @apply .mb-1 .mt-2; /* Trick to have heading scroll into view and not being hidden by header */ &:before { content: ""; - display: block; - @apply .h-3; + @apply hidden .h-3; + } + &:target:before { + @apply block; } .anchor { @apply .hidden; diff --git a/docs/src/styling.md b/docs/src/styling.md index 1471a73a..3c2b453c 100644 --- a/docs/src/styling.md +++ b/docs/src/styling.md @@ -3,6 +3,8 @@ layout: two-columns title: Customize autocomplete styles --- +// TODO + ## Attribution We're happy to provide DocSearch free of charge for your site, and you're diff --git a/docs/src/tips.md b/docs/src/tips.md index 3ee81ade..d50dc9a5 100644 --- a/docs/src/tips.md +++ b/docs/src/tips.md @@ -1,95 +1,79 @@ --- layout: two-columns -title: Recommendations +title: Tips for a good search --- -Here we look at several ingredients that help guarantee best results. -You can also have a look at our [blog that discusses the pitfalls and recommendations](https://blog.algolia.com/how-to-build-a-helpful-search-for-technical-documentation-the-laravel-example/) that create the DocSearch adventure! +DocSearch can work with almost any website, but we found that some site +structure yield more relevant result and/or faster indexing time. In this page +we'll share some tips on how you can make the most out of DocSearch. -## Page Structure +## Use a `sitemap.xml` -For DocSearch to work, your **documentation** must be structured in a certain way. Structure reflects **clarity**, **exactness**, and **steadiness**. A helpful document will always have an understandable structure in one glance. -It allows a user to get the ins and outs of the information and to quickly assimilate the **context**. It will help her/him to establish the **potential relevancy** of the document regarding her/his **intent**. +If your website has a sitemap, DocSearch will use it to get the list of pages to +index. If it can't find one, it will follow every link of every page instead. -Last but not least the structure introduces a lot of meta information that are game changers at indexing time, such as **document hierarchy** or the use of [facets](https://www.algolia.com/doc/guides/searching/faceting/#faceting-overview). +We highly recommend you add a `sitemap.xml` to your website if you don't have +one already. This will make the indexing faster, but will also give you more +control over which page you'd like to include or not in the indexing. -## Document hierarchy +Sitemaps are also considered good practice for other aspects, including SEO +([more information on sitemaps][1]). -As previously mentioned, the **meta information** is almost as important as the payload itself. -The hierarchical path of a document is one of the most important pieces of metadata information. -It brings out the **context** of the document and thus empowers its content with an additional relationship. -(Here we are referring to Custom Ranking). +## Structure the hierarchy of information -Finding the **right depth** of your tree and how to split-up your content is one of the **most complex tasks**. -For large documents, we usually recommend having 4 levels (from lvl0 until lvl3). -Three different levels should be the minimum. +DocSearch works better on structured documentation. Relevance of results is +based on the structural hierarchy of content. In simpler terms it means that we +read the `

`, ..., `

` headings of your page to guess the hierarchy of +information. -The **depth** of a record is also really enlightening. -It will be considered within our [ranking formula](https://www.algolia.com/doc/guides/ranking/ranking-formula/) thanks to the [attributeToIndex](https://www.algolia.com/doc/api-reference/api-parameters/searchableAttributes/) settings. -Definitely the higher up the content is, the more impact it will have. +Documentation starts by explains generic concepts first and then goes deeper +into specifics. This is represented in your HTML markup by the hierarchy of +headings you're using. For example, concepts discussed under a `

` are more +specific than concepts discussed under a `

` in the same page. -In order to deliver the best user experience, it is key to open the page at the exact position of the match. -The location of the match is easier to find in a **fragmented** document thanks to anchors. -This hierarchy must be total and complete in order to be meaningful, -this is why you should try to avoid heterogeneity. +DocSearch uses this structure to fine-tune the relevance of results as well as +to provide potential filtering. Documentation that follow this pattern often +have better relevance of search results. -## Consistency +_Note that you don't have to use `` tags and can use `` for example instead. Your crawling configuration file will + need to mirror those changes, though._ -Consistency is a pillar of a meaningful documentation. In addition to increasing -the **intelligibility** of a document it also shortens the time required for a user -to find the coveted information. The document's **topic** should be easily **identifiable** -and its **outline** sharply demarcated. +## Set a unique class to the element holding the content -The hierarchy should always have the same size. Try to **avoid orphan records** -such like the implicit introduction/conclusion or asides. The selectors must be -efficient for **every document** and highlight the proper hierarchy. -They need to match only the coveted elements depending on their level. -Be careful to avoid the **edge effect** by matching unexpected **superfluous elements**. +As DocSearch is extracting content based on the HTML structure, it is important +to narrow its results to only the relevant content. In that regard, we +recommend that you add a custom `id` to the HTML element that will be the parent +of all your textual content. -We encourage you to build selectors that are **flexible from a DOM structure** point of view -but really **discriminating regarding the documentation's information**. -They should be as much as possible GUI-agnostic and focus on the **characterization of the payload**. +Having such a unique identifier will make your configuration more robust as it +will make sure all content that is indexed is relevant content. We found that +this is the most reliable way to exclude headers, sidebars and footers content +that are not relevant to the search. -Selectors should only match information from **real document webpage** -and remain ineffective for others ones (e.g., landing page, table of content, etc.). -We urge the maintainer to define a **dedicated class** for the **main DOM container** -that includes the actual document content such as `.docSearch-content` +## Add anchors to headings -Since documentation should be **interactive**, it is a key point to -**verbalize concepts with standardized words**. -This **redundancy**, empowered with the **search experience** (dropdown), -will even enable the user to **learn at searching time**. -The **way to find the information** plays a key role in **leading** the user to the -**retrieved knowledge itself**. You can also use the **synonym feature**. +When using headings (as mentioned above), you should also try to add a custom +anchor to each of them. Anchors are HTML attributes (`name` or `id`) +added to headers that will allow the browser to directly scroll to the right +position in the page when clicking a link with a `#` in it. -## Unicity +DocSearch will honor such anchors and automatically bring your users to the +anchor closest to the search result they selected. -The more time-consuming reading documentation is, the more painful and reluctant its use will be. -You must avoid hazy points or catch-all. In addition to it being unhelpful, -the catch-all document may be **confusing** and **counterproductive**. +## Marking the active page(s) in the navigation -Last but not least duplicates introduce noise and mislead users. This is why -you should always focus on the relevant content and avoid duplicating content -within your site (e.g. landing page which contains all of the information, summing up, etc.). -In cases where the duplicates's existence is expected since it belongs to another -dataset (e.g. a different version), you should use [facets](https://www.algolia.com/doc/guides/searching/faceting/). +If you're using a multi-level navigation, we recommend that you mark +each active level with a custom CSS class. This will make it easier for +DocSearch to know _where_ the current page fits in the website +hierarchy. -## Conciseness +For example, if your `troubleshooting.html` page is located under the +`Installation` menu in your sidebar, we recommend that you add a custom CSS +class to the `Installation` and `Troubleshooting` links in your sidebar. -What is clearly thought out is clearly and concisely expressed. +The name of the CSS class does not matter, as long as it's something that can be +used as part of a CSS selector. -## The Crawler, The Discovery Process, Exhaustivity -Since every documentation should be complete, the search experience must **cover the whole of it**. -By default our scraper is crawling your website: it **follows hyperlinks referenced** from the scraped pages. -If these links point to a page which belongs to the allowed domain, -this precise page will be scraped, crawled and so forth. -This **de facto discovery** is really practical but not exhaustive enough. -If for any reason, a webpage could not be referenced from another covered one, -the scope of the search will be imperfect. - -For those reasons we highly recommend that you use a [**Sitemap**](https://www.sitemaps.org/). - -This lists every page of your web site and will be used as the **main source of truth** -and it will define the roadmap of our scraping. -Beside this exhaustivity, using a sitemap introduces a significant performance improvement for our scraper. +[1]: https://www.sitemaps.org/index.html diff --git a/docs/src/what-is-docsearch.md b/docs/src/what-is-docsearch.md index bc02ce0c..0444d0f7 100644 --- a/docs/src/what-is-docsearch.md +++ b/docs/src/what-is-docsearch.md @@ -1,25 +1,27 @@ --- layout: two-columns -title: Overview +title: What is DocSearch? --- -We're scratching our own itch here. As developers, we spend a lot of time -reading documentation, and it isn't always easy to find the information we need. +DocSearch is born out of the need to scratch our own itch. As developers, we +spent a lot of time reading documentation, and we often found it hard to find +relevant information we need quickly. We're not blaming anyone here; building +a good search is a challenge. -Not blaming anyone here. Building a good search for a documentation is a complex -challenge. We happen to have a lot of experience doing that, and we want to -share it with the world. +It just happens that we are a search company and we actually have a lot of +experience building search interfaces. We wanted to put those skills to good +use. That's why we created a way to automatically extract content from tech +documentation and make it available to everyone with only a few keystrokes. +DocSearch itself is made of a crawler and a front-end library. We run the +crawler on our end every 24h to extract content from your website and push it to +an Algolia index. You'll then have to add the front-end library to your website +to redirect all the search requests to this index. -DocSearch will provide, for free and out of the box, the Algolia search -experience for your documentation website. You only need to [ask us to configure -your site](https://community.algolia.com/docsearch/) or -[submit you own configuration](https://github.com/algolia/docsearch-configs). We'll -get back to you with what you need to integrate your new search into your website. +DocSearch is entirely free and mostly automated. The only thing we'll need from +you is to check that you're eligible (most tech projects are), and apply! After +that, we'll share with you the JavaScript snippet needed to add DocSearch to +your website. We just ask that you keep the "powered by Algolia" link displayed. - 1. We'll **configure** your search experience, - 2. We'll **crawl** and **scrap** your documentation pages every 24h accordingly, - 3. You'll only need to **add few lines to your website**: a Javascript snippet along with 2 libraries (JS & CSS). - - **NB:** If you prefer to DIY, you can [run the scraper](https://github.com/algolia/docsearch-scraper) - in your own infra. Feel free to fork! +DocSearch is [one of our ways](https://opencollective.com/algolia) to give back +to the Open-Source community for everything it did for us already.