Refactor isbn api + allow updating with decitre in admin

This commit is contained in:
Gabriel Augendre 2019-06-28 16:33:59 +02:00
parent c2e9b9e86b
commit 9d7b8a1964
4 changed files with 153 additions and 101 deletions

View file

@ -3,7 +3,7 @@ from import_export import resources, fields
from import_export.admin import ExportMixin from import_export.admin import ExportMixin
from import_export.widgets import IntegerWidget, DecimalWidget from import_export.widgets import IntegerWidget, DecimalWidget
from manuels.models import Teacher, Book, Level, Editor, SuppliesRequirement, CommonSupply from manuels.models import Teacher, Book, Level, Editor, SuppliesRequirement, CommonSupply, ISBNError
class TeacherResource(resources.ModelResource): class TeacherResource(resources.ModelResource):
@ -126,6 +126,24 @@ class BookAdmin(ExportMixin, admin.ModelAdmin):
] ]
readonly_fields = ['created_at', 'updated_at'] readonly_fields = ['created_at', 'updated_at']
def update_with_decitre(self, request, queryset):
for book in queryset:
try:
book.update_from_decitre()
messages.success(
request,
f'Mise à jour réussie du livre "{book.title}" ({book.level.name} - {book.field})'
)
except ISBNError as e:
messages.warning(
request,
f'Erreur lors de la mise à jour du livre "{book.title}" ({book.level.name} - {book.field}) : {e.data.get("error")}'
)
update_with_decitre.short_description = 'Mettre à jour avec Decitre'
actions = [update_with_decitre]
@admin.register(Editor) @admin.register(Editor)
class EditorAdmin(admin.ModelAdmin): class EditorAdmin(admin.ModelAdmin):

View file

@ -1,8 +1,10 @@
import logging
import os import os
import re import re
import uuid as uuid import uuid as uuid
import bs4
import requests
from django.conf import settings from django.conf import settings
from django.contrib.postgres.fields import CIEmailField from django.contrib.postgres.fields import CIEmailField
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
@ -12,6 +14,15 @@ from django.db.models import Sum
from django.template.loader import render_to_string from django.template.loader import render_to_string
from django.urls import reverse from django.urls import reverse
from manuels.utils import validate_isbn
logger = logging.getLogger(__name__)
class ISBNError(Exception):
def __init__(self, data):
self.data = data
class BaseModel(models.Model): class BaseModel(models.Model):
class Meta: class Meta:
@ -246,6 +257,102 @@ class Book(BaseModel):
from django.urls import reverse from django.urls import reverse
return reverse('edit_book', kwargs={'teacher_pk': str(self.teacher.pk), 'pk': str(self.pk)}) return reverse('edit_book', kwargs={'teacher_pk': str(self.teacher.pk), 'pk': str(self.pk)})
def update_from_decitre(self):
decitre_data = self.fetch_from_decitre(self.isbn)
self.title = decitre_data.get('title')
self.authors = decitre_data.get('authors')
self.price = decitre_data.get('price')
self.publication_year = decitre_data.get('year')
editor = decitre_data.get('editor')
try:
self.editor = Editor.objects.get(name__icontains=editor)
except Editor.DoesNotExist:
self.editor = Editor.objects.get(name__istartswith='autre')
self.save()
@staticmethod
def fetch_from_decitre(isbn: str):
isbn = isbn.strip().replace('-', '')
if not validate_isbn(isbn):
raise ISBNError({
'error': "L'ISBN saisi n'est pas valide."
})
if len(isbn) == 10:
raise ISBNError({
'error': "La recherche sur Decitre ne fonctionne qu'avec un ISBN 13 (ou EAN)."
})
try:
res = requests.get(f'https://www.decitre.fr/livres/{isbn}.html', timeout=10)
except requests.exceptions.Timeout as exc:
raise ISBNError({
'error': "Decitre n'a pas répondu dans les temps. Message : {}".format(str(exc))
})
try:
res.raise_for_status()
except Exception as exc:
message = (
"Erreur lors de la recherche. Il se peut que le livre n'existe pas dans la base de connaissances "
"de Decitre ou que vous ayez mal saisi l'ISBN. Vous pouvez toujours saisir "
"les informations du livre à la main. Message : {}").format(str(exc))
raise ISBNError({
'error': message
})
decitre_soup = bs4.BeautifulSoup(res.text, "html.parser")
title = decitre_soup.select('h1.product-title')
if title:
title = title[0]
if title.span:
title.span.extract()
title = title.get_text(strip=True)
authors = decitre_soup.select('.authors')
if authors:
authors = authors[0]
authors = authors.get_text(strip=True)
price = decitre_soup.select('div.price span.final-price')
if price:
price = price[0]
price = price.get_text().replace('', '').replace(',', '.').strip()
year = None
editor = None
extra_info = decitre_soup.select('.informations-container')
logger.debug('raw extra_info')
logger.debug(extra_info)
if not extra_info:
logger.debug('#fiche-technique')
extra_info = decitre_soup.select('#fiche-technique')
logger.debug('raw extra_info fiche technique')
logger.debug(extra_info)
if extra_info:
extra_info = extra_info[0].get_text(strip=True)
logger.debug('extra_info')
logger.debug(extra_info)
matches = re.search(
r'Date de parution(?: :)?\d{2}/\d{2}/(?P<year>\d{4})Editeur(?: :)?(?P<editor>.+?)(?:ISBN|Collection)',
extra_info)
if matches:
groups = matches.groupdict()
year = groups.get('year')
editor = groups.get('editor').strip()
return {
'title': title,
'authors': authors,
'isbn': isbn,
'price': float(price) if price else None,
'year': year,
'editor': editor,
}
class SuppliesRequirement(BaseModel): class SuppliesRequirement(BaseModel):
class Meta: class Meta:

21
manuels/utils.py Normal file
View file

@ -0,0 +1,21 @@
def validate_isbn(isbn):
_sum = 0
if len(isbn) == 10:
for i, digit in enumerate(isbn):
if digit == 'X':
digit = 10
else:
digit = int(digit)
_sum += digit * (i + 1)
return _sum % 11 == 0
elif len(isbn) == 13:
for i, digit in enumerate(isbn):
weight = 3 if i % 2 == 1 else 1
digit = int(digit)
_sum += digit * weight
return _sum % 10 == 0
return False

View file

@ -11,7 +11,8 @@ from django.views.decorators.cache import cache_page
from django.views.generic import CreateView, UpdateView, DeleteView, TemplateView from django.views.generic import CreateView, UpdateView, DeleteView, TemplateView
from manuels.forms import AddBookForm, AddSuppliesForm, EditBookForm, EditSuppliesForm from manuels.forms import AddBookForm, AddSuppliesForm, EditBookForm, EditSuppliesForm
from manuels.models import Teacher, Book, SuppliesRequirement, CommonSupply from manuels.models import Teacher, Book, SuppliesRequirement, CommonSupply, ISBNError
from manuels.utils import validate_isbn
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -245,105 +246,10 @@ class ConfirmTeacherView(BaseTeacherView, UpdateView):
return response return response
def validate_isbn(isbn):
_sum = 0
if len(isbn) == 10:
for i, digit in enumerate(isbn):
if digit == 'X':
digit = 10
else:
digit = int(digit)
_sum += digit * (i + 1)
return _sum % 11 == 0
elif len(isbn) == 13:
for i, digit in enumerate(isbn):
weight = 3 if i % 2 == 1 else 1
digit = int(digit)
_sum += digit * weight
return _sum % 10 == 0
return False
# We are able to cache the response because it's very unlikely that the details of a book will change through time # We are able to cache the response because it's very unlikely that the details of a book will change through time
@cache_page(2 * 60 * 60) @cache_page(2 * 60 * 60)
def isbn_api(request, isbn): def isbn_api(request, isbn):
isbn = isbn.strip().replace('-', '')
if not validate_isbn(isbn):
return JsonResponse({
'error': "L'ISBN saisi n'est pas valide."
})
if len(isbn) == 10:
return JsonResponse({
'error': "La recherche sur Decitre ne fonctionne qu'avec un ISBN 13 (ou EAN)."
})
try: try:
res = requests.get(f'https://www.decitre.fr/livres/{isbn}.html', timeout=10) return JsonResponse(Book.fetch_from_decitre(isbn))
except requests.exceptions.Timeout as exc: except ISBNError as e:
return JsonResponse({ return JsonResponse(e.data)
'error': "Decitre n'a pas répondu dans les temps. Message : {}".format(str(exc))
})
try:
res.raise_for_status()
except Exception as exc:
message = ("Erreur lors de la recherche. Il se peut que le livre n'existe pas dans la base de connaissances "
"de Decitre ou que vous ayez mal saisi l'ISBN. Vous pouvez toujours saisir "
"les informations du livre à la main. Message : {}").format(str(exc))
return JsonResponse({
'error': message
})
decitre_soup = bs4.BeautifulSoup(res.text, "html.parser")
title = decitre_soup.select('h1.product-title')
if title:
title = title[0]
if title.span:
title.span.extract()
title = title.get_text(strip=True)
authors = decitre_soup.select('.authors')
if authors:
authors = authors[0]
authors = authors.get_text(strip=True)
price = decitre_soup.select('div.price span.final-price')
if price:
price = price[0]
price = price.get_text().replace('', '').replace(',', '.').strip()
year = None
editor = None
extra_info = decitre_soup.select('.informations-container')
logger.debug('raw extra_info')
logger.debug(extra_info)
if not extra_info:
logger.debug('#fiche-technique')
extra_info = decitre_soup.select('#fiche-technique')
logger.debug('raw extra_info fiche technique')
logger.debug(extra_info)
if extra_info:
extra_info = extra_info[0].get_text(strip=True)
logger.debug('extra_info')
logger.debug(extra_info)
matches = re.search(r'Date de parution(?: :)?\d{2}/\d{2}/(?P<year>\d{4})Editeur(?: :)?(?P<editor>.+?)(?:ISBN|Collection)', extra_info)
if matches:
groups = matches.groupdict()
year = groups.get('year')
editor = groups.get('editor').strip()
return JsonResponse({
'title': title,
'authors': authors,
'isbn': isbn,
'price': float(price) if price else None,
'year': year,
'editor': editor,
})