From 42a4e5676c77721edfa88cbb4c81f7a7eb853232 Mon Sep 17 00:00:00 2001 From: Agie Ashwood Date: Wed, 7 Aug 2024 23:06:15 +0000 Subject: [PATCH] Source addition --- .gitignore | 5 ++ README.md | 13 +++- requirements.txt | 2 + src/Cache.py | 141 ++++++++++++++++++++++++++++++++++ src/Cache.rst | 6 ++ src/Catch.py | 74 ++++++++++++++++++ src/Catch.rst | 5 ++ src/Daisy.py | 195 +++++++++++++++++++++++++++++++++++++++++++++++ src/Daisy.rst | 6 ++ src/Ref.py | 6 ++ src/Soil.py | 49 ++++++++++++ src/Soil.rst | 6 ++ src/Store.py | 33 ++++++++ src/Store.rst | 6 ++ 14 files changed, 546 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 requirements.txt create mode 100755 src/Cache.py create mode 100644 src/Cache.rst create mode 100755 src/Catch.py create mode 100644 src/Catch.rst create mode 100755 src/Daisy.py create mode 100644 src/Daisy.rst create mode 100644 src/Ref.py create mode 100644 src/Soil.py create mode 100644 src/Soil.rst create mode 100755 src/Store.py create mode 100644 src/Store.rst diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d35d22e --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +**/__pycache__/ +bin/ +lib* +share/ +pyvenv.cfg diff --git a/README.md b/README.md index d3d006e..03f3d0a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ # Daisy -Lightweight msgpack based schemaless local and distributed database \ No newline at end of file +Lightweight msgpack based schemaless local and distributed database + +# License + +``` +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with this program. If not, see [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). + +``` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..78eee98 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +msgpack +watchdog diff --git a/src/Cache.py b/src/Cache.py new file mode 100755 index 0000000..161daf4 --- /dev/null +++ b/src/Cache.py @@ -0,0 +1,141 @@ +from Daisy.Daisy import Daisy + +import os + +import msgpack + +from watchdog.observers import Observer + +# TODO: Dumping to cacheFile + + +class Cache: + """ + In memory collection of Daisy records + + `🔗 Source `__ + """ + + def __init__( + self, + filepaths=None, + cacheFile=None, + path: str = "daisy", + walk: bool = False, + isCatch: bool = False, + ): + """ + Parameters + ---------- + filepaths + Either a list of filepaths to load or None + + cacheFile + Path to a cache file which is a collection of paths to load + + path: str + Path prefix to load records from + + walk: bool + Whether to automatically walk the path and load records + + isCatch: bool + Whether this cache is for catchs + """ + self.data = {} + self.path = path + + if filepaths != None: + for fp in filepaths: + fp = path + "/" + fp + if os.path.isfile(fp): + self.data[fp] = Daisy(fp) + elif cacheFile != None: + with open(cacheFile, "r") as f: + for fp in f.read().split("\n"): + self.data[fp] = Daisy(fp) + elif walk: + for root, dirs, files in os.walk(self.path): + for p in dirs + files: + if not (".json" in p): + if not (".md" in p): + tpath = root + "/" + p + self.data[tpath] = Daisy(tpath) + + def create(self, path: str, data: dict, remote=False): + """ + Create new record + + Parameters + ---------- + path: str + Path to create record at + + data: dict + Data to populate record with + """ + if remote == False: + with open(self.path + "/" + path, "wb") as f: + f.write(msgpack.dumps(data)) + # logging.log(10, "Done creating record") + self.data[path] = Daisy(self.path + "/" + path) + # logging.log(10, "Done loading to Daisy") + return self.data[path] + else: + self.data[path] = Ref(path, remote) + return self.data[path] + + def get(self, path: str): + """ + Get record at path, else return False + + path: str + Path of record + """ + if path in self.data.keys(): + return self.data[path] + else: + if os.path.exists(self.path + "/" + path): + self.data[path] = Daisy(self.path + "/" + path) + return self.data[path] + else: + # logging.log(10, "File does not exist") + return False + + def refresh(self): + """ + Reload from disk to memory + """ + for key in self.data.keys(): + self.data[key].read() + + def search(self, keydict: dict, strict: bool = True): + """ + Search cache for record for records with values + + keydict: dict + Values to search for + + strict: bool + Whether to require values match + """ + results = [] + for key, val in self.data.items(): + val = val.get() + if strict and type(val) != str: + addcheck = False + for k, v in keydict.items(): + if k in val.keys(): + if v in val[k]: + addcheck = True + else: + addcheck = False + break + if addcheck: + results.append([key, val]) + elif type(val) != str: + for k, v in keydict.items(): + if k in val.keys(): + if v in val[k]: + results.append([key, val]) + return results diff --git a/src/Cache.rst b/src/Cache.rst new file mode 100644 index 0000000..671a13c --- /dev/null +++ b/src/Cache.rst @@ -0,0 +1,6 @@ +Daisy based cache +================= + +.. autoclass:: Daisy.Cache.Cache + :members: + :undoc-members: diff --git a/src/Catch.py b/src/Catch.py new file mode 100755 index 0000000..79c05c2 --- /dev/null +++ b/src/Catch.py @@ -0,0 +1,74 @@ +from Daisy.Cache import Cache +from Daisy.Ref import Ref + +import os +import random + + +class Catch(Cache): + """ + Sub class of Cache for handling catchs + + .. image:: https://git.utopic.work/PierMesh/piermesh/raw/branch/main/imgs/catchdisplay.png + + `🔗 Source `__ + """ + + catches = {} + remoteCatchesMap = {} + + def __init__( + self, path: str = "catch", filepaths=None, catchFile=None, walk: bool = False + ): + """ + Basically the same initialization parameters as Catch + """ + super().__init__( + filepaths=filepaths, cacheFile=catchFile, path=path, walk=walk, isCatch=True + ) + + # TODO: Fins + + def sget(self, path: str): + """ + Call Cache's get to get record + """ + return super().get(path) + + # TODO: Rename + def get(self, head: str, tail: str, fins=None): + """ + Get catch by pieces + + Parameters + ---------- + head: str + First part of catch (maximum: 4 characters) + + tail: str + Second part of catch (maximum: 16 characters) + + fins + List of (maximum 8 characters) strings at the end of the catch oe None if none + """ + r = self.search({"head": head, "tail": tail}) + return r[0][1]["html"] + + def addc(self, peer, node, seperator, head, tail, data, fins=None, remote=False): + tnpath = "catch/" + node + if os.path.exists(tnpath) != True: + os.makedirs(tnpath) + tppath = tnpath + "/" + peer + if os.path.exists(tppath) != True: + os.makedirs(tppath) + sid = str(random.randrange(0, 999999)).zfill(6) + data["seperator"] = seperator + data["head"] = head + data["tail"] = tail + if fins != None: + data["fins"] = fins + res = self.create("{0}/{1}/{2}".format(node, peer, sid), data, remote=remote) + return [sid, res] + + def exportDirectoryListing(self): + return [k for k in self.data.keys] diff --git a/src/Catch.rst b/src/Catch.rst new file mode 100644 index 0000000..39c3fc6 --- /dev/null +++ b/src/Catch.rst @@ -0,0 +1,5 @@ +Daisy Catch cache +================= + +.. autoclass:: Daisy.Catch.Catch + :members: diff --git a/src/Daisy.py b/src/Daisy.py new file mode 100755 index 0000000..dd2505e --- /dev/null +++ b/src/Daisy.py @@ -0,0 +1,195 @@ +import os +import json +import msgpack + +# TODO: delete +# TODO: propagate json changes to msgpack automatically +# TODO: propagate msgpack changes to cache automatically +# TODO: Indexing + + +def _json_to_msg(path: str): + """ + Convert json at the path plus .json to a msgpack binary + + Parameters + ---------- + path: str + Path to json minus the extension + """ + rpath = path + ".json" + res = b"" + with open(rpath) as f: + res = msgpack.dumps(json.load(f)) + with open(path, "wb") as f: + f.write(res) + + +class Daisy: + """ + Base class for Daisy data representation + + `🔗 Source `__ + + Attributes + ---------- + filepath: str + Path to file representation on disk + + msg: dict + In memory representation + """ + + def __init__( + self, + filepath: str, + templates: dict = {}, + template: bool = False, + prefillDict: bool = False, + remote=False, + ): + """ + Parameters + ---------- + filepath: str + Path to disk location + + templates: dict + Dictionary of templates to Use + + template: bool + Which template to Use + + prefillDict: bool + Whether to fill the record with a template + """ + self.remote = False + self.filepath = filepath + if remote != False: + self.remote = True + self.remoteNodeID = remote + else: + if os.path.exists(filepath) != True: + with open(filepath, "wb") as f: + if template != False: + if template in templates.keys(): + t = templates[template].get() + if prefillDict != False: + for k in prefillDict.keys(): + t[k] = prefillDict[k] + f.write(msgpack.dumps(t)) + self.msg = t + else: + print("No such template as: " + template) + else: + f.write(msgpack.dumps({})) + self.msg = {} + elif os.path.isdir(filepath): + self.msg = "directory" + else: + with open(filepath, "rb") as f: + self.msg = msgpack.loads(f.read()) + + # Use override for updating + + def write( + self, + override=False, + encrypt: bool = False, + encryptKey=None, + recur: bool = False, + ): + """ + Write record to disk + + Parameters + ---------- + override + Either false or a dictionary of values to set on the record + + encrypt: bool + Whether to encrypt the record (TODO) + + encryptKey + Key to encrypt record with, or None if not set + + recur: bool + Whether to recursively handle keys + """ + if override != False: + for key in override.keys(): + # TODO: Deeper recursion + if recur: + if not key in self.msg.keys(): + self.msg[key] = {} + for ikey in override[key].keys(): + self.msg[key][ikey] = override[key][ikey] + else: + self.msg[key] = override[key] + data = msgpack.dumps(self.msg) + with open(self.filepath, "wb") as f: + f.write(data) + + # Use for refreshing + + def read(self, decrypt: bool = False, decryptKey=False): + """ + Read record from disk to memory + + Parameters + ---------- + decrypt: bool + Whether to decrypt record + + decryptKey + Key to decrypt record + """ + if os.path.isdir(self.filepath): + self.msg = "directory" + else: + with open(self.filepath, "rb") as f: + self.msg = msgpack.loads(f.read()) + + def get(self): + """ + Get record dictionary from memory + + Returns + ------- + self.msg: dict + """ + return self.msg + + def sublist(self): + """ + Lists contents of directory if object is a directory, otherwise return None + """ + fpath = self.filepath + if os.path.isdir(fpath): + return ["messages/" + x for x in os.listdir(fpath)] + else: + return None + + +def loadTemplates(templatePath: str = "templates"): + """Load templates for prefilling records + + Parameters + ---------- + templatePath: str + Path to templates + """ + templates = {} + for p in os.listdir(templatePath): + p = templatePath + "/" + p + if os.path.isdir(p): + for ip in os.listdir(p): + ip = p + "/" + ip + if os.path.isdir(ip): + print("Too deep, skipping: " + ip) + else: + templates[ip] = Daisy(ip) + else: + templates[p] = Daisy(p) + self.templates = templates + return templates diff --git a/src/Daisy.rst b/src/Daisy.rst new file mode 100644 index 0000000..d035afe --- /dev/null +++ b/src/Daisy.rst @@ -0,0 +1,6 @@ +Daisy +===== + +.. autoclass:: Daisy.Daisy.Daisy + :members: + :undoc-members: diff --git a/src/Ref.py b/src/Ref.py new file mode 100644 index 0000000..4ec77ca --- /dev/null +++ b/src/Ref.py @@ -0,0 +1,6 @@ +from Daisy.Daisy import Daisy + + +class Ref(Daisy): + def __init__(self, path, remoteNodeID): + super().__init__(path, remote=remoteNodeID) diff --git a/src/Soil.py b/src/Soil.py new file mode 100644 index 0000000..9975d79 --- /dev/null +++ b/src/Soil.py @@ -0,0 +1,49 @@ +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +global garden +""" +Map of instances to list of signals +to be processed +""" +garden = {} + + +class Compound(FileSystemEventHandler): + """ + File system watcher to propagate disk changes + + `🔗 Source `__ + """ + + def __init__(self, cache, isCatch: bool = False): + """ + Parameters + ---------- + cache: Cache + Daisy cache to update + + isCatch: bool + Is the cache for catchs + """ + self.cache = cache + self.isCatch = isCatch + super().__init__() + + def on_any_event(self, event): + """ + Called when a CRUD operation is performed on a record file + + Parameters + ---------- + event + Event object provided by watchdog + """ + if not (".json" in event.src_path): + if not (".md" in event.src_path): + tpath = "/".join(event.src_path.split("/")[1:]) + if tpath != "": + if self.isCatch: + self.cache.sget(tpath) + else: + self.cache.get(tpath).get() diff --git a/src/Soil.rst b/src/Soil.rst new file mode 100644 index 0000000..7005beb --- /dev/null +++ b/src/Soil.rst @@ -0,0 +1,6 @@ +Soil: Daisy signal management +============================= + +.. autoclass:: Daisy.Soil.Compound + :members: + :undoc-members: diff --git a/src/Store.py b/src/Store.py new file mode 100755 index 0000000..deb70e8 --- /dev/null +++ b/src/Store.py @@ -0,0 +1,33 @@ +from Daisy.Daisy import Daisy + +import os + + +class Store(Daisy): + """ + Key value store + + `🔗 Source `__ + """ + + def __init__(self, store: str, path: str, nodeNickname: str): + fpath = "daisy/{0}/{1}".format(path, nodeNickname) + cpath = "{0}/{1}/{2}".format(path, nodeNickname, store) + if not os.path.exists(fpath): + os.mkdir(fpath) + super().__init__("daisy/" + cpath) + + def update(self, entry: str, data, recur: bool = True): + if recur: + for key in data.keys(): + self.msg[entry][key] = data[key] + else: + self.msg[entry] = data + self.write() + + def getRecord(self, key: str): + if key in self.get().keys(): + return self.get()[key] + else: + self.cLog(20, "Record does not exist") + return False diff --git a/src/Store.rst b/src/Store.rst new file mode 100644 index 0000000..6e423d8 --- /dev/null +++ b/src/Store.rst @@ -0,0 +1,6 @@ +Store: Daisy key value store +============================ + +.. autoclass:: Daisy.Store.Store + :members: + :undoc-members: