From 4f2e505de00fae635563a7b0b78b38ea03485033 Mon Sep 17 00:00:00 2001 From: Jun Wei Woon Date: Mon, 6 Jul 2020 13:36:54 +0800 Subject: [PATCH] Added robots.txt --- config/_default/config.toml | 3 +++ content/en/posts/directory_cleanup.md | 2 ++ layouts/robots.txt | 11 +++++++++++ 3 files changed, 16 insertions(+) create mode 100644 layouts/robots.txt diff --git a/config/_default/config.toml b/config/_default/config.toml index b9524b6..bb4de01 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -6,6 +6,9 @@ defaultContentLanguage = "en" defaultContentLanguageInSubdir = true hasCJKLanguage = true +# Robots.txt +enableRobotsTXT = true + summaryLength = 70 buildFuture = true diff --git a/content/en/posts/directory_cleanup.md b/content/en/posts/directory_cleanup.md index b6b670f..2aa81b1 100644 --- a/content/en/posts/directory_cleanup.md +++ b/content/en/posts/directory_cleanup.md @@ -31,7 +31,9 @@ A clean home/project directory allows us to be able to work productively, withou This is a simple script to help you clean your directory in seconds ### Caution +{{< alert theme="danger" dir="ltr" >}} __DO NOT__ run the script on any directory containing system/important files. +{{< /alert >}} ## Dependencies * [Python3.8](https://www.python.org/) diff --git a/layouts/robots.txt b/layouts/robots.txt new file mode 100644 index 0000000..f7b9f94 --- /dev/null +++ b/layouts/robots.txt @@ -0,0 +1,11 @@ +User-agent: * +Allow: / + +User-agent: * +Disallow: /gallery + +User-agent: * +Disallow: /static + +User-agent: * +Disallow: /images