From 2c435c6199f80a21f573cfeeafa2730af1431b00 Mon Sep 17 00:00:00 2001 From: Gabriel Augendre Date: Sat, 29 May 2021 15:57:49 +0200 Subject: [PATCH] Implement API to add new short urls --- .envrc.dist | 7 +- poetry.lock | 33 +++++++- pyproject.toml | 2 + shortener/{ovh => api_connectors}/__init__.py | 0 .../api_connectors/base_api_connector.py | 6 ++ shortener/api_connectors/ovh/__init__.py | 0 shortener/api_connectors/ovh/ovh_api.py | 38 +++++++++ shortener/cache.py | 8 ++ shortener/data_structures.py | 13 +++ shortener/exceptions.py | 4 + shortener/handlers.py | 83 ++++++++++++++----- shortener/main.py | 37 +++++---- shortener/ovh/ovh_api.py | 2 - shortener/resolver.py | 26 ++++++ shortener/templates/base.html | 11 +++ shortener/templates/not_found.html | 7 ++ shortener/templates/result.html | 22 +++++ shortener/templates/welcome.html | 14 ++++ 18 files changed, 272 insertions(+), 41 deletions(-) rename shortener/{ovh => api_connectors}/__init__.py (100%) create mode 100644 shortener/api_connectors/base_api_connector.py create mode 100644 shortener/api_connectors/ovh/__init__.py create mode 100644 shortener/api_connectors/ovh/ovh_api.py create mode 100644 shortener/cache.py create mode 100644 shortener/data_structures.py delete mode 100644 shortener/ovh/ovh_api.py create mode 100644 shortener/resolver.py create mode 100644 shortener/templates/base.html create mode 100644 shortener/templates/not_found.html create mode 100644 shortener/templates/result.html create mode 100644 shortener/templates/welcome.html diff --git a/.envrc.dist b/.envrc.dist index a20a829..4863cc7 100644 --- a/.envrc.dist +++ b/.envrc.dist @@ -1,3 +1,8 @@ export FLASK_APP=shortener.main export FLASK_ENV=development -export BASE_DOMAIN=g4b.ovh. +export BASE_DOMAIN=g4b.ovh +export PROVIDER=ovh +export OVH_ENDPOINT=ovh-eu +export OVH_APPLICATION_KEY= +export OVH_APPLICATION_SECRET= +export OVH_CONSUMER_KEY= diff --git a/poetry.lock b/poetry.lock index dc650d7..87c7ac2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -161,6 +161,18 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "ovh" +version = "0.5.0" +description = "\"Official OVH.com API wrapper\"" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +dev = ["coverage (==3.7.1)", "mock (==1.0.1)", "nose (==1.3.3)", "yanc (==0.2.4)", "Sphinx (==1.2.2)", "coveralls (==0.4.4)", "setuptools (>=30.3.0)", "wheel"] +test = ["coverage (==3.7.1)", "mock (==1.0.1)", "nose (==1.3.3)", "yanc (==0.2.4)"] + [[package]] name = "packaging" version = "20.9" @@ -245,6 +257,17 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +[[package]] +name = "redis" +version = "3.5.3" +description = "Python client for Redis key-value store" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +hiredis = ["hiredis (>=0.1.3)"] + [[package]] name = "six" version = "1.16.0" @@ -301,7 +324,7 @@ watchdog = ["watchdog"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "dfd3993450db4fbb1625a378f1c1018769d937dfc7175ef464474ac9388deef9" +content-hash = "7bb0d58e52c792760a354cbff8e7ae8b642d5d29fabf3210a1b32d1f5e025a81" [metadata.files] appdirs = [ @@ -400,6 +423,10 @@ nodeenv = [ {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, ] +ovh = [ + {file = "ovh-0.5.0-py2.py3-none-any.whl", hash = "sha256:03c7a5e7a62e7bc09b899f7692c694360be7db93ebe44428d6605fccda244692"}, + {file = "ovh-0.5.0.tar.gz", hash = "sha256:f74d190c4bff0953d76124cb8ed319a8a999138720e42957f0db481ef4746ae8"}, +] packaging = [ {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"}, {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"}, @@ -447,6 +474,10 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] +redis = [ + {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"}, + {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyproject.toml b/pyproject.toml index 385ef58..e74530a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,8 @@ authors = ["Gabriel Augendre "] python = "^3.9" dnspython = "^2.1.0" Flask = "^2.0.0" +ovh = "^0.5.0" +redis = "^3.5.3" [tool.poetry.dev-dependencies] pytest = "^5.2" diff --git a/shortener/ovh/__init__.py b/shortener/api_connectors/__init__.py similarity index 100% rename from shortener/ovh/__init__.py rename to shortener/api_connectors/__init__.py diff --git a/shortener/api_connectors/base_api_connector.py b/shortener/api_connectors/base_api_connector.py new file mode 100644 index 0000000..eaaf647 --- /dev/null +++ b/shortener/api_connectors/base_api_connector.py @@ -0,0 +1,6 @@ +from shortener.data_structures import UserInput + + +class BaseApiConnector: + def add_record(self, user_input: UserInput) -> None: + raise NotImplementedError("Subclasses must implement add_record") diff --git a/shortener/api_connectors/ovh/__init__.py b/shortener/api_connectors/ovh/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shortener/api_connectors/ovh/ovh_api.py b/shortener/api_connectors/ovh/ovh_api.py new file mode 100644 index 0000000..f66108c --- /dev/null +++ b/shortener/api_connectors/ovh/ovh_api.py @@ -0,0 +1,38 @@ +import os + +import ovh +from ovh import HTTPError, InvalidResponse + +from shortener.api_connectors.base_api_connector import BaseApiConnector +from shortener.data_structures import UserInput +from shortener.exceptions import ApiException + + +class OvhApi(BaseApiConnector): + def __init__(self): + endpoint = os.environ["OVH_ENDPOINT"] + application_key = os.environ["OVH_APPLICATION_KEY"] + application_secret = os.environ["OVH_APPLICATION_SECRET"] + consumer_key = os.environ["OVH_CONSUMER_KEY"] + self._base_domain = os.environ["BASE_DOMAIN"] + self._client = ovh.Client( + endpoint=endpoint, + application_key=application_key, + application_secret=application_secret, + consumer_key=consumer_key, + ) + + def add_record(self, user_input: UserInput) -> None: + try: + self._add_record(user_input) + except (HTTPError, InvalidResponse) as e: + raise ApiException(message=str(e)) + + def _add_record(self, user_input: UserInput) -> None: + self._client.post( + f"/domain/zone/{self._base_domain}/record", + fieldType="TXT", + subDomain=user_input.short_code, + target=user_input.url, + ) + self._client.post(f"/domain/zone/{self._base_domain}/refresh") diff --git a/shortener/cache.py b/shortener/cache.py new file mode 100644 index 0000000..b9ec040 --- /dev/null +++ b/shortener/cache.py @@ -0,0 +1,8 @@ +import os + +import redis + +REDIS_HOST = os.getenv("REDIS_HOST", "localhost") +REDIS_PORT = int(os.getenv("REDIS_PORT", 6379)) +REDIS_DB = int(os.getenv("REDIS_DB", 0)) +cache = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB) diff --git a/shortener/data_structures.py b/shortener/data_structures.py new file mode 100644 index 0000000..38738cf --- /dev/null +++ b/shortener/data_structures.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + +from flask import request + + +@dataclass +class UserInput: + short_code: str + url: str + + @property + def short_url(self): + return f"{request.host_url}{self.short_code}" diff --git a/shortener/exceptions.py b/shortener/exceptions.py index 5211c90..f795091 100644 --- a/shortener/exceptions.py +++ b/shortener/exceptions.py @@ -7,3 +7,7 @@ class ApiException(Exception): class BadRequestException(ApiException): def __init__(self, message="Bad request", code=400): super().__init__(message, code) + + +class ShortCodeNotFoundException(Exception): + pass diff --git a/shortener/handlers.py b/shortener/handlers.py index 7291a18..64b047a 100644 --- a/shortener/handlers.py +++ b/shortener/handlers.py @@ -1,27 +1,72 @@ +import os +from typing import Dict +from urllib.parse import urlparse + from flask import request -from shortener.exceptions import BadRequestException -from shortener.ovh.ovh_api import OvhApi +from shortener.api_connectors.base_api_connector import BaseApiConnector +from shortener.api_connectors.ovh.ovh_api import OvhApi +from shortener.cache import cache +from shortener.data_structures import UserInput +from shortener.exceptions import BadRequestException, ShortCodeNotFoundException +from shortener.resolver import resolver -SUPPORTED_PROVIDERS = {"ovh": OvhApi} - - -def handle_expand(shortcode, resolver): - answer = resolver.resolve(shortcode, "TXT", search=True, lifetime=10) - redirect_url = list(answer)[0].strings[0].decode("utf-8") - return redirect_url +SUPPORTED_PROVIDERS: Dict[str, type] = {"ovh": OvhApi} +PROVIDER_NAME = os.environ["PROVIDER"].lower() +PROVIDER_CLASS = SUPPORTED_PROVIDERS[PROVIDER_NAME] +COOLDOWN = "cooldown" def handle_shorten(): - user_input = request.json - print(user_input) - provider = user_input.get("provider") - if not provider: - raise BadRequestException(message="`provider` is required") - provider_class = SUPPORTED_PROVIDERS.get(provider) - if not provider_class: - supported_providers = list(SUPPORTED_PROVIDERS.keys()) + user_input: UserInput = _validate_user_input() + cooldown = cache.get(user_input.short_code) + if cooldown: raise BadRequestException( - message=f"`provider` is not supported. Must be one of {supported_providers}" + message=f"Please wait before submitting again for {user_input.short_code}." ) - return {"status": "ok"} + provider: BaseApiConnector = _get_provider() + provider.add_record(user_input) + cache.set(user_input.short_code, COOLDOWN, ex=90) + return {"status": "ok", "url": user_input.short_url} + + +def _validate_user_input(): + user_input = request.json + if not user_input: + user_input = request.form + url = user_input.get("url") + if not url: + raise BadRequestException(message="`url` is required") + parsed_url = urlparse(url) + if not all([parsed_url.scheme, parsed_url.netloc]): + raise BadRequestException( + message="`url` doesn't seem well formatted. " + "Please include at least protocol and domain name. " + "E.g. 'https://google.com' instead of 'google.com'." + ) + + short_code = user_input.get("short_code") + if not short_code: + raise BadRequestException(message="`short_code` is required") + short_code = str(short_code) + if len(short_code) < 4: + raise BadRequestException( + message="`short_code` must be longer than 4 characters." + ) + if _short_code_exists(short_code): + raise BadRequestException(message=f"`{short_code}` is already taken") + + user_input = UserInput(short_code=short_code, url=url) + return user_input + + +def _get_provider() -> BaseApiConnector: + return PROVIDER_CLASS() + + +def _short_code_exists(short_code): + try: + resolver.resolve_shortcode(short_code) + return True + except ShortCodeNotFoundException: + return False diff --git a/shortener/main.py b/shortener/main.py index 6a01eb6..93fcfb0 100644 --- a/shortener/main.py +++ b/shortener/main.py @@ -1,39 +1,40 @@ import os -from dns.name import from_text -from dns.resolver import LRUCache, Resolver -from flask import Flask, redirect +from flask import Flask, redirect, render_template, request -from shortener.exceptions import ApiException -from shortener.handlers import handle_expand, handle_shorten - -BASE_DOMAIN = os.getenv("BASE_DOMAIN") +from shortener.exceptions import ApiException, ShortCodeNotFoundException +from shortener.handlers import handle_shorten +from shortener.resolver import resolver app = Flask(__name__) -resolver = Resolver() -resolver.search = [from_text(BASE_DOMAIN)] -resolver.cache = LRUCache() @app.route("/") def home(): - return "

Welcome to the url shortener!

" + return render_template("welcome.html") -@app.route("/shorten/", methods=["POST"]) +@app.route("/shorten", methods=["POST"]) def shorten_url(): try: - return handle_shorten() + result = handle_shorten() + code = 200 except ApiException as e: - return {"status": "error", "message": e.message}, e.code + result = {"message": e.message, "status": "error"} + code = e.code + is_api = bool(request.json) + if is_api: + return result, code + + return render_template("result.html", **result), code -@app.route("//") +@app.route("/") def expand_url(shortcode: str): try: - redirect_url = handle_expand(shortcode, resolver) - except: - return "

URL not found

" + redirect_url = resolver.resolve_shortcode(shortcode) + except ShortCodeNotFoundException: + return render_template("not_found.html") return redirect(redirect_url, code=302) diff --git a/shortener/ovh/ovh_api.py b/shortener/ovh/ovh_api.py deleted file mode 100644 index 8827c68..0000000 --- a/shortener/ovh/ovh_api.py +++ /dev/null @@ -1,2 +0,0 @@ -class OvhApi: - pass diff --git a/shortener/resolver.py b/shortener/resolver.py new file mode 100644 index 0000000..5ddf54f --- /dev/null +++ b/shortener/resolver.py @@ -0,0 +1,26 @@ +import os + +from dns.name import from_text +from dns.resolver import LRUCache, Resolver + +from shortener.exceptions import ShortCodeNotFoundException + +BASE_DOMAIN = os.getenv("BASE_DOMAIN") + + +class ShortenerResolver(Resolver): + def __init__(self): + super().__init__() + self.search = [from_text(BASE_DOMAIN + ".")] + self.cache = LRUCache() + + def resolve_shortcode(self, shortcode): + try: + answer = self.resolve(shortcode, "TXT", search=True, lifetime=10) + redirect_url = list(answer)[0].strings[0].decode("utf-8") + return redirect_url + except: + raise ShortCodeNotFoundException() + + +resolver = ShortenerResolver() diff --git a/shortener/templates/base.html b/shortener/templates/base.html new file mode 100644 index 0000000..bd2ba12 --- /dev/null +++ b/shortener/templates/base.html @@ -0,0 +1,11 @@ + + + + + URL shortener + + +{% block content %} +{% endblock %} + + diff --git a/shortener/templates/not_found.html b/shortener/templates/not_found.html new file mode 100644 index 0000000..f4a5f55 --- /dev/null +++ b/shortener/templates/not_found.html @@ -0,0 +1,7 @@ +{% extends "base.html" %} + +{% block content %} +

URL not found

+

If you recently created the redirect, please wait a couple of minutes for DNS to propagate.

+

If you've been sent this URL, please contact the sender. It's likely there's an issue with the URL you received.

+{% endblock %} diff --git a/shortener/templates/result.html b/shortener/templates/result.html new file mode 100644 index 0000000..b97fe78 --- /dev/null +++ b/shortener/templates/result.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} + +{% block content %} +

Result

+

Status: {{ status }}

+ {% if message %} +

{{ message }}

+ {% endif %} + {% if url %} +

+ Your short URL is: {{ url }}.
+ Please wait a couple of minutes for DNS to propagate before testing and + sharing this URL. +

+ {% endif %} +

+ + Go back + +

+{% endblock %} diff --git a/shortener/templates/welcome.html b/shortener/templates/welcome.html new file mode 100644 index 0000000..924cc10 --- /dev/null +++ b/shortener/templates/welcome.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block content %} +

Welcome!

+

Welcome to the URL shortener.

+
+ + + + + + +
+{% endblock %}