Update robots.txt

raselshikdar · Oct 5, 2024 · 11e4c67 · 11e4c67
1 parent 00a8bf2
commit 11e4c67
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,2 +1,16 @@
 User-agent: *
 Allow: /
+
+Disallow: /wp-admin/
+Disallow: /cgi-bin/
+Disallow: /search
+
+# Disallow access to sensitive files
+Disallow: /private/
+Disallow: /temp/
+
+# Allow all crawlers to index sitemap
+Sitemap: https://info.rasel.us.kg/sitemap.xml
+
+# Optional: Specify crawl delay to prevent server overload
+Crawl-delay: 10