From 9beffc9e39f6007df60bd25767f126b21fd21e88 Mon Sep 17 00:00:00 2001 From: Suwako Moriya Date: Sat, 21 Mar 2020 21:15:02 +0100 Subject: [PATCH] [errors, perdu, storage] From Pickle to JSON --- modules/errors/__init__.py | 25 ++++++++++++------------- modules/perdu/__init__.py | 8 ++++---- storage/base.py | 30 ++++++++++++++---------------- storage/jsonenc.py | 20 ++++++++++++++++++++ 4 files changed, 50 insertions(+), 33 deletions(-) create mode 100644 storage/jsonenc.py diff --git a/modules/errors/__init__.py b/modules/errors/__init__.py index a9d9033..541bf04 100644 --- a/modules/errors/__init__.py +++ b/modules/errors/__init__.py @@ -1,5 +1,4 @@ import asyncio -import collections import random import traceback @@ -26,24 +25,24 @@ class MainClass(BaseClassPython): def __init__(self, client): super().__init__(client) self.config.init({"dev_chan": [], "memes": [""], "icon": ""}) - self.errorsDeque = None + self.errorsList = None async def on_load(self): - if await self.objects.save_exists('errorsDeque'): - self.errorsDeque = await self.objects.load_object('errorsDeque') + if await self.objects.save_exists('errorsList'): + self.errorsList = await self.objects.load_object('errorsList') else: - self.errorsDeque = collections.deque() + self.errorsList = [] async def on_ready(self): - for i in range(len(self.errorsDeque)): + for i in range(len(self.errorsList)): try: - msg_id = self.errorsDeque.popleft() + msg_id = self.errorsList.pop(0) channel = self.client.get_channel(msg_id["channel_id"]) to_delete = await channel.fetch_message(msg_id["msg_id"]) await to_delete.delete() except: raise - await self.objects.save_object('errorsDeque', self.errorsDeque) + await self.objects.save_object('errorsList', self.errorsList) async def command(self, message, args, kwargs): raise Exception("KERNEL PANIC!!!") @@ -86,9 +85,9 @@ class MainClass(BaseClassPython): message = await channel.send(embed=embed.set_footer(text="Ce message va s'autodétruire dans une minute", icon_url=self.config.icon)) msg_id = {"channel_id": message.channel.id, "msg_id": message.id} - self.errorsDeque.append(msg_id) - # Save message in errorsDeque now to keep them if a reboot happend during next 60 seconds - self.objects.save_object('errorsDeque', self.errorsDeque) + self.errorsList.append(msg_id) + # Save message in errorsList now to keep them if a reboot happend during next 60 seconds + await self.objects.save_object('errorsList', self.errorsList) # Wait 60 seconds and delete message await asyncio.sleep(60) @@ -100,8 +99,8 @@ class MainClass(BaseClassPython): raise finally: try: - self.errorsDeque.remove(msg_id) + self.errorsList.remove(msg_id) except ValueError: pass # Save now to avoid deleting unkown message - self.objects.save_object('errorsDeque', self.errorsDeque) + await self.objects.save_object('errorsList', self.errorsList) diff --git a/modules/perdu/__init__.py b/modules/perdu/__init__.py index 4ebf202..0a04ee3 100644 --- a/modules/perdu/__init__.py +++ b/modules/perdu/__init__.py @@ -23,11 +23,11 @@ class MainClass(BaseClassPython): def __init__(self, client): super().__init__(client) - self.config.init({"channel": 0, "lost_role": 0, "min_delta": datetime.timedelta(minutes=30).total_seconds()}) + self.config.init({"channel": 0, "lost_role": 0, "min_delta": datetime.timedelta(minutes=26).total_seconds()}) async def on_load(self): if await self.objects.save_exists('history'): - self.history = self.objects.load_object('history') + self.history = await self.objects.load_object('history') else: self.history = {} @@ -44,7 +44,7 @@ class MainClass(BaseClassPython): delta = message.created_at - self.history[message.author.id][-1][0] if delta.total_seconds() >= self.config.min_delta: self.history[message.author.id].append((message.created_at, delta)) - self.objects.save_object("history", self.history) + await self.objects.save_object("history", self.history) await self.parse_command(message) async def fill_history(self): @@ -58,7 +58,7 @@ class MainClass(BaseClassPython): delta = self.history[message.author.id][-1][0] - message.created_at if delta.total_seconds() >= self.config.min_delta: self.history[message.author.id].append((message.created_at, delta)) - self.objects.save_object("history", self.history) + await self.objects.save_object("history", self.history) def get_top(self, top=10, since=datetime.datetime(year=1, month=1, day=1)): """Return [(userid, [(date, delta), (date,delta), ...]), ... ]""" diff --git a/storage/base.py b/storage/base.py index 1be2e92..a868e6e 100644 --- a/storage/base.py +++ b/storage/base.py @@ -1,5 +1,5 @@ -import pickle - +import json +import storage.jsonenc from storage import path as pth @@ -125,19 +125,17 @@ class Objects: self.storage = storage self.storage.makedirs("objects", exist_ok=True) - def save_object(self, object_name, object_instance): - """Save object into pickle file""" - with self.storage.open(pth.join("objects", object_name), "wb") as f: - pickler = pickle.Pickler(f) - pickler.dump(object_instance) + async def save_object(self, object_name, object_instance): + """Save object into json file""" + with self.storage.open(pth.join("objects", object_name + ".json"), "w") as f: + json.dump([object_instance], f, cls=storage.jsonenc.Encoder) - def load_object(self, object_name): - """Load object from pickle file""" - if self.save_exists(object_name): - with self.storage.open(pth.join("objects", object_name), "rb") as f: - unpickler = pickle.Unpickler(f) - return unpickler.load() + async def load_object(self, object_name): + """Load object from json file""" + if await self.save_exists(object_name): + with self.storage.open(pth.join("objects", object_name + ".json"), "r") as f: + return json.load(f, object_hook=storage.jsonenc.hook)[0] - def save_exists(self, object_name): - """Check if pickle file exists""" - return self.storage.exists(pth.join("objects", object_name)) + async def save_exists(self, object_name): + """Check if json file exists""" + return self.storage.exists(pth.join("objects", object_name + ".json")) diff --git a/storage/jsonenc.py b/storage/jsonenc.py new file mode 100644 index 0000000..40ebd64 --- /dev/null +++ b/storage/jsonenc.py @@ -0,0 +1,20 @@ +import json +import datetime + + +class Encoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime.datetime)): + return {'data_type':'datetime.datetime', 'iso':obj.isoformat()} + if isinstance(obj, (datetime.timedelta)): + return {'data_type':'datetime.timedelta', 'totalseconds':obj.total_seconds()} + return json.JSONEncoder.default(self, obj) + + +def hook(dct): + if 'data_type' in dct: + if dct['data_type'] == "datetime.datetime": + return datetime.datetime.fromisoformat(dct['iso']) + elif dct['data_type'] == "datetime.timedelta": + return datetime.timedelta(seconds=dct['totalseconds']) + return dct