diff --git a/ChangeLog b/ChangeLog index a741498..8079ed7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,5 @@ -v0.2 (08/11/2015) +v0.2 (17/01/2016) **User** Add autofocus to login page + Add search diff --git a/denote/models.py b/denote/models.py index 8c7852b..47de3eb 100644 --- a/denote/models.py +++ b/denote/models.py @@ -24,8 +24,12 @@ import re from django.db import models from django.contrib.auth.models import AbstractUser from django.http import HttpResponse +from django.db.models.signals import pre_init, post_init, pre_delete, post_delete +from django.db.models.signals import pre_save, post_save +from django.dispatch import receiver import markdown2 +import search class User(AbstractUser): hidden_categories = models.TextField(blank=True) @@ -113,8 +117,20 @@ class Note(models.Model): self.modified_date = datetime.now() self.transformed_text = markdown2.markdown(self.text, extras=['fenced-code-blocks']) self._summarize() + s = Search() + super(Note, self).save() +@receiver(post_save, sender=Note) +def post_save_note_signal(sender, **kwargs): + s = Search() + s.edit_note(kwargs['instance'].id) + +@receiver(pre_delete, sender=Note) +def pre_delete_note_signal(sender, **kwargs): + s = Search() + s.remove_note(kwargs['instance'].id) + def manage_category(user, cat_name): category = None if cat_name: diff --git a/denote/search.py b/denote/search.py new file mode 100755 index 0000000..d48cf67 --- /dev/null +++ b/denote/search.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +""" + Copyright 2016 Grégory Soutadé + + This file is part of Dénote. + + Dynastie is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Dynastie is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Dynastie. If not, see . +""" +import re +import unicodedata +import os +import operator +import pickle +from django.db import models + +import models +#from models import Note + +class Search: + MINIMUM_LETTERS = 3 + + def __init__(self): + self.report = '' + + self.tagreg = re.compile('<[^>]+>') + self.htmlreg = re.compile('&[^;]+;') + self.numreg = re.compile('[0-9]+') + self.pat = re.compile(r'\s+') + + self.replace_by_space = (u'(', u')', u'#', u'\'', u'{', u'}', u'[', u']', + u'-', u'|', u'\t', u'\\', u'_', u'^' '=', u'+', u'$', + u'£', u'%', u'µ', u'*', u',', u'?', u';', u'.', u'/', + u':', u'!', u'§', u'€', u'²') + + # Imported from generator.py + def _addReport(self, string, color=''): + if color != '': + self.report = self.report + '' + self.report = self.report + '' + self.__class__.__name__ + ' : ' + self.report = self.report + string + if color != '': + self.report = self.report + '' + self.report = self.report + '
\n' + + def _addWarning(self, string): + self.addReport(string, 'yellow') + + def _addError(self, string): + self.addReport(string, 'red') + + + def _saveDatabase(self, hashtable): + d = pickle.dumps(hashtable) + + f = open(os.environ['DENOTE_ROOT'] + '/_search.db', 'w') + f.write(d) + f.close() + + def _loadDatabase(self): + filename = os.environ['DENOTE_ROOT'] + '/_search.db' + + if not os.path.exists(filename): + print 'No search index !' + return {} + + f = open(filename, 'rb') + hashtable = pickle.load(f) + f.close() + + return hashtable + + def _strip_accents(self, s): + return ''.join((c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')) + + def _remove_tag(self, content): + content = self.htmlreg.sub('', content) + content = self.numreg.sub('', content) + + content = content.replace('\n', '') + content = content.replace('\r', '') + content = content.replace('"', '') + + for c in self.replace_by_space: + content = content.replace(c, ' ') + + content = self.tagreg.sub('', content) + + content = self.pat.sub(' ', content) + + return content + + def _prepare_string(self, content): + content = self._remove_tag(content) + content = self._strip_accents(content) + + return content + + def _indexContent(self, hashtable, index, content, word_weight): + content = self._prepare_string(content) + + wordlist = content.split(' ') + + for word in wordlist: + if len(word) < self.MINIMUM_LETTERS: + continue + word = word.lower() + if not word in hashtable: + hashtable[word] = [] + if not index in hashtable[word]: + hashtable[word].insert(0, [index, word_weight]) + else: + weight = hashtable[word][1] + hashtable[word][1] = weight + word_weight + + def _index(self, hashtable, index): + try: + note = Note.objects.get(pk=index) + except: + return + + self._indexContent(hashtable, index, note.text, 1) + self._indexContent(hashtable, index, note.title.encode('utf-8'), 5) + + def _index_note(self, note, saveDatabase=True): + hashtable = self._loadDatabase() + + self._index(hashtable, int(note)) + + if saveDatabase: + self._saveDatabase(hashtable) + + def _remove_note(self, note, saveDatabase=True): + hashtable = self._loadDatabase() + + if hashtable is None: return + + for k, v in hashtable.items(): + # For tuples in values + for t in v: + if note == v[0]: + v.remove(t) + + if saveDatabase: + self._saveDatabase(hashtable) + + def generate_index(self, notes): + hashtable = self._loadDatabase() + + for note in notes: + self._indexContent(hashtable, note.id, note.text, 1) + self._indexContent(hashtable, note.id, note.title, 5) + + self._saveDatabase(hashtable) + + def index_note(self, note): + return self._index_note(note, True) + + def delete_note(self, note): + return self._remove_note(note, True) + + def edit_note(self, note, saveDatabase=True): + self._remove_note(note, False) + self._index_note(note, True) + + def search(self, string): + hashtable = self._loadDatabase() + + string = self._prepare_string(string.encode('utf-8')) + + wordlist = string.split(' ') + + res = {} + for word in wordlist: + if len(word) < Search.MINIMUM_LETTERS: + continue + word = word.lower() + reg = re.compile('.*' + word + '.*') + for key in hashtable.keys(): + if reg.match(key): + for note in hashtable[key]: + res[note[0]] = res.get(note[0],0) + note[1] + + sorted_res = sorted(res.iteritems(), key=operator.itemgetter(1)) + sorted_res.reverse() + + res = [sorted_res[i][0] for i in range(len(sorted_res))] + + return res diff --git a/denote/templates/base.html b/denote/templates/base.html index ed34974..57e7c98 100644 --- a/denote/templates/base.html +++ b/denote/templates/base.html @@ -8,7 +8,8 @@ -
Settings Disconnect
+
Settings Disconnect
+
{% csrf_token %}


diff --git a/denote/templates/base_user.html b/denote/templates/base_user.html index 2a1165c..1b02154 100644 --- a/denote/templates/base_user.html +++ b/denote/templates/base_user.html @@ -14,7 +14,8 @@ - +
Settings Disconnect

+
{% csrf_token %}


diff --git a/denote/urls.py b/denote/urls.py index 11d8960..f0b35fd 100644 --- a/denote/urls.py +++ b/denote/urls.py @@ -30,4 +30,6 @@ urlpatterns = patterns('', url(r'^note/(\d+)$', 'denote.views.note', name='note'), url(r'^category/edit/(\d)$','denote.views.edit_category', name='edit_category'), url(r'^preferences$', 'denote.views.preferences', name='preferences'), + url(r'^search$', 'denote.views.search', name='search'), + url(r'^generate_search_index$', 'denote.views.generate_search_index', name='generate_search_index'), ) diff --git a/denote/views.py b/denote/views.py index 6cf8df4..93b73c5 100644 --- a/denote/views.py +++ b/denote/views.py @@ -18,6 +18,7 @@ along with Dénote. If not, see . """ +import os from datetime import datetime from django.http import HttpResponseRedirect, HttpResponse, Http404 @@ -27,6 +28,7 @@ from django.shortcuts import render from denote.models import * from denote.forms import * +from denote.search import * def index(request): if request.user.is_authenticated(): @@ -238,3 +240,33 @@ def preferences(request): else: raise Http404 +@login_required +def search(request): + context = _prepare_note_context(request.user) + + ref = request.META['HTTP_REFERER'] + + if 'text' in request.POST: + text = request.POST['text'] + else: + return HttpResponseRedirect(ref) + + s = Search() + note_list = s.search(text) + + notes = Note.objects.filter(pk__in=note_list, author=request.user) + context['notes'] = notes + context['note_form'] = NoteForm() + + return render(request, 'user_index.html', context) + +@login_required +def generate_search_index(request): + + if os.path.exists('_search.db'): + os.path.remove('_search.db') + + s = Search() + s.generate_index(Note.objects.all()) + + return HttpResponseRedirect('/')