Add FileOutputCache to avoid disk reading generated posts if not necessary

Always write post on edit (don't try to hash)
This commit is contained in:
Gregory Soutade 2014-06-01 11:28:24 +02:00
parent 798b5e1f92
commit ddf267e9ca
3 changed files with 58 additions and 31 deletions

View File

@ -90,19 +90,35 @@ class DynastieGenerator:
return int(res)
def writeIfNotTheSame(self, filename, node):
from dynastie.models import FileOutputCache
writer = StrictUTF8Writer()
node.writexml(writer)
content = writer.getvalue().encode('utf-8')
if os.path.exists(filename):
src_md5 = hashlib.md5()
f = open(filename,'rb')
src_md5.update(f.read())
f.close()
dst_md5 = hashlib.md5()
dst_md5.update(content)
if src_md5.digest() == dst_md5.digest():
cache_objs = FileOutputCache.objects.filter(name=filename)
if cache_objs.count() == 0:
cache_obj = None
else:
cache_obj = cache_objs[0]
if not cache_obj is None or os.path.exists(filename):
if cache_obj is None:
src_md5 = hashlib.md5()
f = open(filename,'rb')
src_md5.update(f.read())
f.close()
src_md5 = src_md5.hexdigest()
else:
src_md5 = cache_obj.hash
if src_md5 == dst_md5.hexdigest():
if cache_obj is None:
cache_obj = FileOutputCache(name=filename, hash=src_md5)
cache_obj.save()
filename = filename + '.gz'
if not os.path.exists(filename):
f = gzip.open(filename, 'wb')
@ -111,6 +127,11 @@ class DynastieGenerator:
return
os.unlink(filename)
if cache_obj is None:
cache_obj = FileOutputCache(name=filename, hash=dst_md5.hexdigest())
else:
cache_obj.hash = dst_md5.hexdigest()
self.addReport('Write (and compress) ' + filename)
f = open(filename,'wb')
f.write(content)
@ -122,6 +143,8 @@ class DynastieGenerator:
f.write(content)
f.close()
cache_obj.save()
self.somethingWrote = True
def createLinkElem(self, dom, path, title):

View File

@ -305,38 +305,20 @@ class Post(models.Model):
self.tags.remove(t)
def createPost(self, content, tags):
b = self.blog
output = b.src_path
output = self.blog.src_path
if not os.path.exists(output + '/_post'):
os.mkdir(output + '/_post')
filename = output + '/_post/' + str(self.pk)
content = unicode(content)
content = content.encode('utf-8')
modif = True
if os.path.exists(filename):
f = open(filename, 'rb')
src_md5 = hashlib.md5()
src_md5.update(f.read())
f.close()
dst_md5 = hashlib.md5()
dst_md5.update(content)
if src_md5.digest() == dst_md5.digest():
modif = False
else:
os.unlink(filename)
if modif:
f = open(filename, 'wb')
f.write(content)
f.close()
self.modification_date=datetime.now()
self.manageTags(tags)
self.save()
def remove(self):
b = self.blog
@ -432,6 +414,10 @@ class Comment(models.Model):
def _remove_br(self):
self.the_comment = self.the_comment.replace('<br />', '\n')
class FileOutputCache(models.Model):
name = models.CharField(max_length=512)
hash = models.CharField(max_length=512)
@receiver(post_init, sender=Blog)
def init_blog_signal(sender, **kwargs):
kwargs['instance'].create_paths()

View File

@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<ljdc>
<entry>
<id>87094581962</id>
<address>http://thecodinglove.com/post/87094581962/when-caffeine-is-no-longer-effective</address>
<title>when caffeine is no longer effective</title>
<img>http://i.imgur.com/ocjPFJL.gif</img>
</entry>
<entry>
<id>85721632915</id>
<address>http://lesjoiesducode.fr/post/85721632915/quand-je-suggere-lusage-de-nouvelles-technos</address>
<title>quand je suggère l'usage de nouvelles technos</title>
<img>http://i.imgur.com/BezORLq.gif</img>
</entry>
<entry>
<id>85709017865</id>
<address>http://lesjoiesducode.fr/post/85709017865/quand-un-collegue-me-refile-un-ticket-facile-a</address>
<title>quand un collègue me refile un ticket "facile à traiter"</title>
<img>http://ljdchost.com/XyeioZc.gif</img>
</entry>
<entry>
<id>85704348740</id>
<address>http://lesjoiesducode.fr/post/85704348740/quand-le-client-reclame-une-feature-hors-cahier-des</address>