Add a robots.txt

articles/context_processors.py

@@ -1,13 +1,23 @@
 from articles.models import Article, Page
 
+IGNORED_PATHS = [
+    "/robots.txt",
+]
+
 
 def pages(request):
+    if request.path in IGNORED_PATHS:
+        return {}
     return {"pages": Page.objects.filter(status=Article.PUBLISHED)}
 
 
 def drafts_count(request):
+    if request.path in IGNORED_PATHS:
+        return {}
     return {"drafts_count": Article.objects.filter(status=Article.DRAFT).count()}
 
 
 def date_format(request):
+    if request.path in IGNORED_PATHS:
+        return {}
     return {"CUSTOM_ISO": r"Y-m-d\TH:i:sO"}

attachments/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

blog/settings.py

@@ -88,7 +88,7 @@ ROOT_URLCONF = "blog.urls"
 TEMPLATES = [
     {
         "BACKEND": "django.template.backends.django.DjangoTemplates",
-        "DIRS": [],
+        "DIRS": ["blog/templates"],
         "APP_DIRS": True,
         "OPTIONS": {
             "context_processors": [

blog/templates/blog/robots.txt

@@ -0,0 +1,2 @@
+User-Agent: *
+Disallow: /admin/

blog/tests/test_robots.py

@@ -0,0 +1,6 @@
+def test_robots_txt(client):
+    res = client.get("/robots.txt")
+    assert res.status_code == 200
+    assert res["Content-Type"] == "text/plain"
+    content = res.content.decode("utf-8")
+    assert "User-Agent" in content

blog/urls.py

@@ -16,11 +16,18 @@ Including another URLconf
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import path
+from django.views.generic import TemplateView
 
 from articles.views import feeds, html
 from blog import settings
 
 urlpatterns = [
+    path(
+        "robots.txt",
+        TemplateView.as_view(
+            template_name="blog/robots.txt", content_type="text/plain"
+        ),
+    ),
     path("admin/", admin.site.urls),
     path("", html.ArticlesListView.as_view(), name="articles-list"),
     path("drafts/", html.DraftsListView.as_view(), name="drafts-list"),