From b44e7e5c7113b5dfa3ab40f9d84c6790ea93d533 Mon Sep 17 00:00:00 2001 From: Pixelastic Date: Mon, 13 Aug 2018 18:26:48 +0200 Subject: [PATCH] Rewriting the essentials --- docs/config.json | 12 ++-- docs/src/_includes/apply-form.pug | 19 ++++++ docs/src/_layouts/homepage.pug | 19 +----- docs/src/_layouts/two-columns.pug | 6 +- docs/src/apply.md | 33 ++--------- docs/src/config-file.md | 6 ++ docs/src/faq.md | 97 +++++++++++++++++++++++-------- docs/src/inside-the-engine.md | 48 +++++++++++++++ docs/src/integrations.md | 26 +++++++++ docs/src/who-can-apply.md | 54 +++++++++++++++++ 10 files changed, 243 insertions(+), 77 deletions(-) create mode 100644 docs/src/_includes/apply-form.pug create mode 100644 docs/src/integrations.md create mode 100644 docs/src/who-can-apply.md diff --git a/docs/config.json b/docs/config.json index ca483b9f..1de8987d 100644 --- a/docs/config.json +++ b/docs/config.json @@ -18,10 +18,18 @@ "title": "How does it work?", "url": "how-does-it-work.html" }, + { + "title": "Who can apply?", + "url": "who-can-apply.html" + }, { "title": "Apply now", "url": "apply.html" }, + { + "title": "Integrations", + "url": "integrations.html" + }, { "title": "Tips", "url": "tips.html" @@ -29,10 +37,6 @@ { "title": "FAQ", "url": "faq.html" - }, - { - "title": "FAQ2", - "url": "faq2.html" } ] }, diff --git a/docs/src/_includes/apply-form.pug b/docs/src/_includes/apply-form.pug new file mode 100644 index 00000000..ffcec338 --- /dev/null +++ b/docs/src/_includes/apply-form.pug @@ -0,0 +1,19 @@ +form.m-auto.max-w-sm.elevation.p-2.md_p-3 + .mb-2 + label.control-label(for="url") Website + input#url.form-control(name="url", placeholder="https://project.org/doc") + p.text-grey-dark.text-xs.italic We’ll crawl the documentation pages of your website and index them on Algolia. + .mb-2 + label.control-label(for="email") Email + input#email.form-control(name="email", placeholder="you@project.org", type="email") + p.text-grey-dark.text-xs.italic We’ll email you the code you need to integrate the search into your documentation. + .mb-2 + label + input(name="owner", type="checkbox") + span I'm the owner of that website + .mb-2 + p.leading-2 Refer to + a.clickable(href="https://www.algolia.com/policies/terms") Algolia's Privacy Policy + | for more information on how we use and protect your data + button.c-button.c-button-primary.w-100(type="submit") Join the program + diff --git a/docs/src/_layouts/homepage.pug b/docs/src/_layouts/homepage.pug index df0ccd6b..eaa36454 100644 --- a/docs/src/_layouts/homepage.pug +++ b/docs/src/_layouts/homepage.pug @@ -154,24 +154,7 @@ block content .text-5.normal.pb-2.text-white.text-center Get started .pb-1.text-3.leading-2.text-white.text-center We’ll get back to you within a few hours with everything you need to integrate your new search into your website. .pb-3.text-3.leading-2.text-white.text-center Oh, and did we mention it's FREE? No commitment. No subscription. Everything is on us! - form.m-auto.max-w-sm.elevation.p-2.md_p-3 - .mb-2 - label.control-label(for="url") Website - input#url.form-control(name="url", placeholder="https://project.org/doc") - p.text-grey-dark.text-xs.italic We’ll crawl the documentation pages of your website and index them on Algolia. - .mb-2 - label.control-label(for="email") Email - input#email.form-control(name="email", placeholder="you@project.org", type="email") - p.text-grey-dark.text-xs.italic We’ll email you the code you need to integrate the search into your documentation. - .mb-2 - label - input(name="owner", type="checkbox") - span I'm the owner of that website - .mb-2 - p.leading-2 Refer to - a.clickable(href="https://www.algolia.com/policies/terms") Algolia's Privacy Policy - | for more information on how we use and protect your data - button.c-button.c-button-primary.w-100(type="submit") Join the program + include ../_includes/apply-form.pug .text-center.p-2.pb-4 p.text-2.mb-2.leading-2 Docsearch 2015-2018 • Designed and built with love by diff --git a/docs/src/_layouts/two-columns.pug b/docs/src/_layouts/two-columns.pug index bb379a01..74580510 100644 --- a/docs/src/_layouts/two-columns.pug +++ b/docs/src/_layouts/two-columns.pug @@ -24,4 +24,8 @@ block content a.block.py-0x.text-telluric(href=`${page.url}#${heading.anchor}`)=heading.title .documentation-content.fln.w-md.text-2.pb-2.pr-1.sm_px-2.overflow-hidden - !=current.content + !=current.content + // Include the apply form + if current.includeForm + .mt-2 + include ../_includes/apply-form.pug diff --git a/docs/src/apply.md b/docs/src/apply.md index 7885308b..de62d6c9 100644 --- a/docs/src/apply.md +++ b/docs/src/apply.md @@ -1,36 +1,11 @@ --- layout: two-columns title: Applying to DocSearch +includeForm: true --- -// TODO + Thanks for you interest in DocSearch! Please make sur you [read the + checklist][1] before applying. -## Our policy -We really want to improve documentation with search. Finding the information you are looking for is the very first step when reading docs. Search is key. - -We are always really sorry to have to turn down request, but we are receiving so many applications everyday that we want to give some priority to public technical documentation websites. - -Please note that you can [always run DocSearch on your own infra](https://community.algolia.com/docsearch/documentation/docsearch-scraper/overview/) and [use our free community plan](https://www.algolia.com/pricing). We would be happy to [enhance DocSearch with your feedback](https://github.com/algolia/docsearch). - -## What is required? - -If you want us to host everything for you, there are some few requirements you still need to comply with: -- Be **an owner** of the snippet. You will need to inject some JS snippets. -- Your website must be **publicly available**. If your website is not, you will need to run our scrapper on a whitelisted server or implemnet your own authentification strategy (not supported so far). -- Your website is a **documentation**. We will reduce the scope of our scrap to helpful pages only. -- Your website is **production ready**. We will not accept lorem ipsum website. - -## What is considered as a bonus/accelerator - -Since we need to prioritize requests, the following points are big advantages for your project: -- You are **Open-Source** or you provide a free plan. We care about the developer community. -- You are using a **supported documentation generator**. You can find example below. -- Your site is **server-side rendered** (ie no extra JS rendering needed) -- You are already **implementing [the generic configuration](#)** - -We are always happy to help product we love. How could you know if you qualify? [Apply!](https://community.algolia.com/docsearch/) - -## My project is already featured - -If your project is already featured or or uses a **supported** documentation-generator tool (e.g. [pkgdown](http://pkgdown.r-lib.org/index.html), [docusaurus](https://docusaurus.io/), [gitbook](https://www.gitbook.com/), [vuepress](https://vuepress.vuejs.org/) .... DocSearch is totally hosted on our side and stays free and will continue this way. We do recommend you to use [the featured Search-UI](https://github.com/algolia/docsearch) or fork it on your own. +[1]: who-can-apply.html diff --git a/docs/src/config-file.md b/docs/src/config-file.md index c739943b..ca0bdc96 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -25,6 +25,12 @@ Once you run the DocSearch scraper on a specific configuration, it will: - extract the content of every single crawled page following the logic you defined using the CSS selectors - push the resulting records to the Algolia index you configured + + +### Does the crawl encompass several domains/sub-domains? + +The `start_urls` define the allowed domain for our crawler. Basically, we take the main domain of every URLs. We will not go outside this whitelisted domain list. If you want to encompass a wider domain, please include it as a new start_url. + ## Configuration format A configuration file looks like: diff --git a/docs/src/faq.md b/docs/src/faq.md index 7ab1af25..ab751bdf 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -1,46 +1,93 @@ --- layout: two-columns -title: Global wonderings +title: FAQ --- -// TODO +If you're not finding the answer to your question in the website, this +page will help you. If you're still unsure, don't hesitate to send [your +question to us][1] directly. -## What is the general behaviour? +## How often will you crawl my website? -### How often do you scrap my website? +Every day. -We scrap your website, at least, every 24 hours since documentation is not meant to change frequently. -Furthermore we update it after every pull request on your configuration. This is the best way to request us to trigger a new crawl. Please leverage your configuration file to do so. +The exact time of day might vary each day, but we'll crawl your +website at most every 24 hours. We will also trigger a manual crawling every time +your config is updated. -### I want to add search to my whole website? -DocSearch is suited for documentation content. If you want to widen the scope of your search to not documentation content. You have two solutions: -- [Run it on your own](https://community.algolia.com/docsearch/documentation/docsearch-scraper/overview/) -- Build your own search-UI [thanks to InstanteSearch](https://community.algolia.com/instantsearch.js/) ([example here](https://jsfiddle.net/s_pace/965a4w3o/)) +## What do I need to install on my side? -### Does the crawl encompass several domains/sub-domains? +Nothing. -The `start_urls` define the allowed domain for our crawler. Basically, we take the main domain of every URLs. We will not go outside this whitelisted domain list. If you want to encompass a wider domain, please include it as a new start_url. +The DocSearch crawler is running on our own infra. It will read HTML +content from your website and populate an Algolia index with it every day. All +you need to do is keep your website online, and we take care of the rest. -### Where is hosted the data? +## How much does it cost? -As every Algolia index, everything is stored on our servers with the security and privacy required. You can [find more details in the global documentation pages.](https://www.algolia.com/doc/guides/infrastructure/servers/) +Nothing. -## Feature +We know that paying for search infrastructure is a cost not many Open Source +projects can afford. That's why we decided to keep DocSearch free for everyone. +All we ask in exchange is that you keep the _powered by +[Algolia][2]_ logo displayed next to the search results. -### Will it become paid? +If this is not possible for you, you're totally free to open your own Algolia +account and run DocSearch on your own without this limitation. In that case +though, depending on the size of your documentation, you might need a paid +account (free accounts can hold as much as 10k records). -Open source is great and we want to support as much as we can. Since these projects mostly have limited resources, every granted project will ever remain free. -If you want to upgrade your search experience, you can do it with your own Algolia implementation and [apply for a free community plan](https://www.algolia.com/pricing). DocSearch will not be used anymore but still happy to help. +## What data are you collecting? -### What data are you collecting, who can get access to it? +The data we save in the index is the data we extract from your website +markup, in a custom JSON format instead of HTML. Exactly which part of the +page is defined in your custom config. -We only scrap publicly available data according to your custom selectors (see the how it works part). DocSearch introduces [the algolia analytics](https://www.algolia.com/doc/guides/insights-and-analytics/analytics-overview/) for your DocSearch indices. -You can ask access to these data by using the private email thread and send us the email addresses of the people to grant access. +As the website owner, we also give you access to the Algolia Analytics +dashboard, that will let you have more data about the anonymized searches that +were done on your website. You'll be able to see the most searched terms, or +those with no results. -### Where can I see the analytics? +## Where is my data hosted? -Once you are granted (see above), analytics will be available in the Algolia dashboard like any regular application. You will need to select the analytics tabs. +All DocSearch data is hosted on Algolia's servers, with replication around the +globe. You can find more details about the actual [server specs here][3], and more +complete information in our [privacy policy][4]. -## I am not the owner, what can i do? +## Can I use DocSearch on non-doc pages? -You can definitely help improving the search experience of your favourite documentation website by opening issues on the repo. You can also reference the GitHub handle @s-pace if you want us to chime in / provide you a quick demo. You can advocate the owner of the repo by letting them request DocSearch. We would be happy to support you in such regards. Feel free to ping. +The free DocSearch we provide will only crawl documentation pages. If you want +to use it on other parts of your website, you'll need to create your own Algolia +account and either: + +- Run the [DocSearch crawler][5] on your own +- Use one of our other [framework integrations or API clients][6] + +## Can you index code samples? + +Yes, but we do not recommend it. + +Code samples are a great way for humans to understand how a specific pattern +/ method should be used. Unfortunately, it often requires boilerplate code, +repeated into several examples, which will only add noise to the results. + +What we recommend instead is to exclude the code blocks from the indexing (by +using the `selectors_exclude` option in your config), and instead structure your +content so the method names are actual headers. + +## My favorite project documentation does not use DocSearch. What can I do? + +We'd love to help! + +If one of your favorite tool documentation is missing +DocSearch, we encourage you to file an issue in their repository explaining how +DocSearch could help. Feel free to ping `@docsearch` on GitHub in the issue as +well, and we'll provide all the help we can. + + +[1]: mailto:docsearch@algolia.com +[2]: https://www.algolia.com/ +[3]: https://www.algolia.com/doc/guides/infrastructure/servers/ +[4]: https://www.algolia.com/policies/privacy +[5]: ./crawler-overview.html +[6]: https://www.algolia.com/doc/api-reference/ diff --git a/docs/src/inside-the-engine.md b/docs/src/inside-the-engine.md index 3e8d0c1f..ee803406 100644 --- a/docs/src/inside-the-engine.md +++ b/docs/src/inside-the-engine.md @@ -19,3 +19,51 @@ This construction is based on their **time of appearance** along the flow. _**Note:** The above process performs sanity tests as it scrapes, in order to detect errors. If indeed there are any serious warnings, it will abort and therefore not overwrite your current index. These checks ensure that your dedicated index isn't flushed._ + + + +## How are my DocSearch records ranked? + +DocSearch empowers the Algolia ranking strategy. The formula is completely +based on [the tie-breaking approach](https://www.algolia.com/doc/guides/ranking/ranking-formula/#tie-breaking-approach). + +The special feature of DocSearch's ranking resides in [the custom ranking](https://www.algolia.com/doc/guides/ranking/custom-ranking/): + +We have defined 3 main weight indicators for every record. These values are +ordered by importance following the tie-breaking approach: +1. `page_rank`: this value, equal to `0` by default, can be set from the +`start_urls` object. It can be customized in order to boost or restrain some +records depending on **their webpage's URL**. It will need to match a **specific +regular expression pattern**. + +**Example:** + ```json +{ + "index_name": "example", + "start_urls": [ + { + "url": "http://example.com/docs/api/v1\\.[0-9]", + "page_rank": 1 + } + ] +} +``` +2. `level`: this value depends on the `level` of the record. A record's level is +its **deepest level attribute not null**. `text` records have a weight of 0. +3. `position`: This value is the position of the matching element within every +picked up element along the original HTML flow. The **sooner** the record appears, +the **higher** it will be ranked. + +You can override the way these elements are impacting the search thanks to `custom_settings`. + + **Example:** + + ```json +"custom_settings": { + "customRanking": [ + "asc(weight.position)", + "desc(weight.page_rank)", + "desc(weight.level)" + ] +} + ``` diff --git a/docs/src/integrations.md b/docs/src/integrations.md new file mode 100644 index 00000000..403e49e5 --- /dev/null +++ b/docs/src/integrations.md @@ -0,0 +1,26 @@ +--- +layout: two-columns +title: Supported Integrations +--- + +We worked with documentation website generators to have DocSearch directly +embedded as a first class citizen in website they produce. If you're using one +of the following tools, checkout their documentation to see how to enable +DocSearch on your website: + +- [Docusaurus][1] - [How to enable search][2] +- [VuePress][3] - [Algolia Search][4] +- [GitBook][5] +- [pkgdown][6] - [DocSearch indexing][7] + +If you're a maintaining such a tool and would like to get added to the list, get +[in touch with us][8] + +[1]: https://docusaurus.io/ +[2]: https://docusaurus.io/docs/en/search#docsNav +[3]: https://vuepress.vuejs.org/ +[4]: https://vuepress.vuejs.org/default-theme-config/#algolia-search +[5]: https://docs.gitbook.com/ +[6]: http://pkgdown.r-lib.org/index.html +[7]: http://pkgdown.r-lib.org/articles/pkgdown.html#search +[8]: mailto:docsearch@algolia.com diff --git a/docs/src/who-can-apply.md b/docs/src/who-can-apply.md new file mode 100644 index 00000000..a7c53f90 --- /dev/null +++ b/docs/src/who-can-apply.md @@ -0,0 +1,54 @@ +--- +layout: two-columns +title: Who can apply? +--- + +DocSearch has been built from the ground up with the idea of improving search on +large technical documentation. For that reason, we only provide the free hosting +version to technical documentation websites. + +We're always sad to have to turn down applications, but with the number of +requests we received everyday, we had to focus on technical documentations. +We hope you understand. + +## The checklist + +To have your request validated, please make sure you comply with the following +points: + +- You must be the **owner** of the website, or at least have the power to update + its content. You'll have to include a JavaScript snippet to enable DocSearch. +- Your website must be **publicly available**. We cannot index websites that are + sitting behind an authentication or available on your machine. +- Your website must be a **documentation website**. We do not index blogs and + commercial one-pagers. +- Your website must **have some content**. We won't index empty websites nor those + filled with lorem ipsum placeholder content. Please, wait until you have + written some documentation before applying. + +If in doubt, don't hesitate to [apply][1] and we'll figure it out together. + +Even if we cannot accept your request, this does not mean that you cannot enjoy +great search on your website. DocSearch is entirely open-source and you can run +it yourself, or use any of our other API clients to take advantage of the +features of Algolia. + +## Priority + +We're receiving many requests every day, and while we strive to answer them all +as fast as we can, we sometimes give priority to some of them based on the +following criteria: + +- 🙂 If your project is Open-Source, we'll handle it before any other + close-source product. We care about the OSS community and want to help as much + as we can. +- 🙂 If you're using one of our [official integrations][2], creating your config + will be much faster for us. +- ☹️ If your website is rendered in the browser through JavaScript, it means + that we'll have to crawl it through a browser emulation which is much slower + than a typical crawl. We highly recommend that you implement server-side + rendering if you can. + + +[1]: ./apply.html +[2]: ./integrations.html