Browse Source

daily journal temporary storage in a file

tags/1.0
Y 2 years ago
parent
commit
03b36437f0
3 changed files with 63 additions and 40 deletions
  1. +1
    -1
      TODO.md
  2. +60
    -38
      pyruse/actions/action_dailyReport.py
  3. +2
    -1
      tests/main.py

+ 1
- 1
TODO.md View File

@@ -2,7 +2,7 @@

* Insert the GPL stuff in the source files.
* Create a filter that rejects all messages that match a series of regular expressions.
* Handle OTHER messages in a file in order to avoid filling the RAM waiting for the report to be sent.
* Maybe switch from storing the daily journal in a file, to storing it in a database.
* Write the systemd service that starts pyruse on boot.
* Write the systemd service+timer that restores bans after a reboot.
* Maybe switch from Step.run() recursion to Step.run()-in-a-loop to avoid too-deep call stacks.


+ 60
- 38
pyruse/actions/action_dailyReport.py View File

@@ -1,14 +1,17 @@
import datetime
import json
import os
import string
from pyruse import base, email
from collections import OrderedDict
from datetime import datetime
from pyruse import base, config, email

class Action(base.Action):
WARN = "WARN"
INFO = "INFO"
OTHER = "OTHER"
_storage = config.Config().asMap().get("storage", "/var/lib/pyruse") \
+ "/" + os.path.basename(__file__) + ".journal"

_messages = None
_hour = 0
_out = open(_storage, "w", 1)
_out.write("[\n")

_txtDocStart = '= Pyruse Report\n\n'
_txtHeadWarn = '== WARNING Messages\n\n'
@@ -28,23 +31,15 @@ class Action(base.Action):
_htmPreStart = '<pre>'
_htmPreStop = '</pre>\n'

def _encode(self, text):
return text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')

def _toAdoc(self, msg, times):
return "\n|{count:^5d}|{text}\n |{times}\n".format_map(
{"count": len(times), "text": msg, "times": " +\n ".join(str(t) for t in times)}
)

def _toHtml(self, msg, times):
return "<tr><td>{count}</td><td>{text}</td><td>{times}</td></tr>\n".format_map(
{"count": len(times), "text": self._encode(msg), "times": "<br/>".join(str(t) for t in times)}
)

def __init__(self, args):
super().__init__()
self.level = args["level"]
self.isOther = self.level == Action.OTHER
l = args["level"]
if l == "WARN":
self.level = 1
elif l == "INFO":
self.level = 2
else:
self.level = 0
self.template = args["message"]
values = {}
for (_void, name, _void, _void) in string.Formatter().parse(self.template):
@@ -53,28 +48,61 @@ class Action(base.Action):
self.values = values

def act(self, entry):
messages = Action._messages[self.level]
for (name, _void) in self.values.items():
self.values[name] = entry.get(name, None)
msg = self.template.format_map(self.values)
if self.isOther:
messages.append((entry["__REALTIME_TIMESTAMP"], msg))
elif msg in messages:
messages[msg].append(entry["__REALTIME_TIMESTAMP"])
else:
messages[msg] = [entry["__REALTIME_TIMESTAMP"]]
thisHour = datetime.datetime.today().hour
json.dump(
OrderedDict(L = self.level, T = entry["__REALTIME_TIMESTAMP"].timestamp(), M = msg),
Action._out
)
Action._out.write(",\n")
thisHour = datetime.today().hour
if thisHour < Action._hour:
self._closeJournal()
self._sendReport()
self._openJournal()
Action._hour = thisHour

def _closeJournal(self):
Action._out.write("{}]")
Action._out.close()

def _openJournal(self):
Action._out = open(Action._storage, "w", 1)
Action._out.write("[\n")

def _encode(self, text):
return text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')

def _toAdoc(self, msg, times):
return "\n|{count:^5d}|{text}\n |{times}\n".format_map(
{"count": len(times), "text": msg, "times": " +\n ".join(str(t) for t in times)}
)

def _toHtml(self, msg, times):
return "<tr><td>{count}</td><td>{text}</td><td>{times}</td></tr>\n".format_map(
{"count": len(times), "text": self._encode(msg), "times": "<br/>".join(str(t) for t in times)}
)

def _sendReport(self):
messages = [[], {}, {}]
with open(Action._storage) as journal:
for e in json.load(journal):
if e != {}:
(L, T, M) = (e["L"], datetime.fromtimestamp(e["T"]), e["M"])
if L == 0:
messages[0].append((T, M))
elif M in messages[L]:
messages[L][M].append(T)
else:
messages[L][M] = [T]

html = Action._htmDocStart + Action._htmHeadWarn
text = Action._txtDocStart + Action._txtHeadWarn

text += Action._txtTableDelim + Action._txtTableHeader
html += Action._htmTableStart
for (msg, times) in sorted(Action._messages[Action.WARN].items(), key = lambda i: i[0]):
for (msg, times) in sorted(messages[1].items(), key = lambda i: i[0]):
text += self._toAdoc(msg, times)
html += self._toHtml(msg, times)
text += Action._txtTableDelim
@@ -85,7 +113,7 @@ class Action(base.Action):

text += Action._txtTableDelim + Action._txtTableHeader
html += Action._htmTableStart
for (msg, times) in sorted(Action._messages[Action.INFO].items(), key = lambda i: i[0]):
for (msg, times) in sorted(messages[2].items(), key = lambda i: i[0]):
text += self._toAdoc(msg, times)
html += self._toHtml(msg, times)
text += Action._txtTableDelim
@@ -96,7 +124,7 @@ class Action(base.Action):

text += Action._txtPreDelim
html += Action._htmPreStart
for (time, msg) in Action._messages[Action.OTHER]:
for (time, msg) in messages[0]:
m = '%s: %s\n' % (time, msg)
text += m
html += self._encode(m)
@@ -105,9 +133,3 @@ class Action(base.Action):
html += Action._htmDocStop

email.Mail(text, html).send()
Action._messages = _resetMessages()

def _resetMessages():
return {Action.WARN: {}, Action.INFO: {}, Action.OTHER: []}

Action._messages = _resetMessages()

+ 2
- 1
tests/main.py View File

@@ -101,10 +101,11 @@ def main():
"diff -U0 \"$0\"{,.test_ref} | grep -vE '^[-+@^]{2,3} |={5,}[0-9]+=='",
f],
check = True)
assert false, "differences found in " + f
assert False, "differences found in " + f
except subprocess.CalledProcessError:
pass # OK, no difference found
_clean()
os.remove('action_dailyReport.py.journal')

def entry(host, service, message, microsecond = None):
global _microsec


Loading…
Cancel
Save