daily journal temporary storage in a file
parent
fc864feca4
commit
03b36437f0
2
TODO.md
2
TODO.md
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
* Insert the GPL stuff in the source files.
|
* Insert the GPL stuff in the source files.
|
||||||
* Create a filter that rejects all messages that match a series of regular expressions.
|
* Create a filter that rejects all messages that match a series of regular expressions.
|
||||||
* Handle OTHER messages in a file in order to avoid filling the RAM waiting for the report to be sent.
|
* Maybe switch from storing the daily journal in a file, to storing it in a database.
|
||||||
* Write the systemd service that starts pyruse on boot.
|
* Write the systemd service that starts pyruse on boot.
|
||||||
* Write the systemd service+timer that restores bans after a reboot.
|
* Write the systemd service+timer that restores bans after a reboot.
|
||||||
* Maybe switch from Step.run() recursion to Step.run()-in-a-loop to avoid too-deep call stacks.
|
* Maybe switch from Step.run() recursion to Step.run()-in-a-loop to avoid too-deep call stacks.
|
||||||
|
|
|
@ -1,14 +1,17 @@
|
||||||
import datetime
|
import json
|
||||||
|
import os
|
||||||
import string
|
import string
|
||||||
from pyruse import base, email
|
from collections import OrderedDict
|
||||||
|
from datetime import datetime
|
||||||
|
from pyruse import base, config, email
|
||||||
|
|
||||||
class Action(base.Action):
|
class Action(base.Action):
|
||||||
WARN = "WARN"
|
_storage = config.Config().asMap().get("storage", "/var/lib/pyruse") \
|
||||||
INFO = "INFO"
|
+ "/" + os.path.basename(__file__) + ".journal"
|
||||||
OTHER = "OTHER"
|
|
||||||
|
|
||||||
_messages = None
|
|
||||||
_hour = 0
|
_hour = 0
|
||||||
|
_out = open(_storage, "w", 1)
|
||||||
|
_out.write("[\n")
|
||||||
|
|
||||||
_txtDocStart = '= Pyruse Report\n\n'
|
_txtDocStart = '= Pyruse Report\n\n'
|
||||||
_txtHeadWarn = '== WARNING Messages\n\n'
|
_txtHeadWarn = '== WARNING Messages\n\n'
|
||||||
|
@ -28,6 +31,46 @@ class Action(base.Action):
|
||||||
_htmPreStart = '<pre>'
|
_htmPreStart = '<pre>'
|
||||||
_htmPreStop = '</pre>\n'
|
_htmPreStop = '</pre>\n'
|
||||||
|
|
||||||
|
def __init__(self, args):
|
||||||
|
super().__init__()
|
||||||
|
l = args["level"]
|
||||||
|
if l == "WARN":
|
||||||
|
self.level = 1
|
||||||
|
elif l == "INFO":
|
||||||
|
self.level = 2
|
||||||
|
else:
|
||||||
|
self.level = 0
|
||||||
|
self.template = args["message"]
|
||||||
|
values = {}
|
||||||
|
for (_void, name, _void, _void) in string.Formatter().parse(self.template):
|
||||||
|
if name:
|
||||||
|
values[name] = None
|
||||||
|
self.values = values
|
||||||
|
|
||||||
|
def act(self, entry):
|
||||||
|
for (name, _void) in self.values.items():
|
||||||
|
self.values[name] = entry.get(name, None)
|
||||||
|
msg = self.template.format_map(self.values)
|
||||||
|
json.dump(
|
||||||
|
OrderedDict(L = self.level, T = entry["__REALTIME_TIMESTAMP"].timestamp(), M = msg),
|
||||||
|
Action._out
|
||||||
|
)
|
||||||
|
Action._out.write(",\n")
|
||||||
|
thisHour = datetime.today().hour
|
||||||
|
if thisHour < Action._hour:
|
||||||
|
self._closeJournal()
|
||||||
|
self._sendReport()
|
||||||
|
self._openJournal()
|
||||||
|
Action._hour = thisHour
|
||||||
|
|
||||||
|
def _closeJournal(self):
|
||||||
|
Action._out.write("{}]")
|
||||||
|
Action._out.close()
|
||||||
|
|
||||||
|
def _openJournal(self):
|
||||||
|
Action._out = open(Action._storage, "w", 1)
|
||||||
|
Action._out.write("[\n")
|
||||||
|
|
||||||
def _encode(self, text):
|
def _encode(self, text):
|
||||||
return text.replace('&', '&').replace('<', '<').replace('>', '>')
|
return text.replace('&', '&').replace('<', '<').replace('>', '>')
|
||||||
|
|
||||||
|
@ -41,40 +84,25 @@ class Action(base.Action):
|
||||||
{"count": len(times), "text": self._encode(msg), "times": "<br/>".join(str(t) for t in times)}
|
{"count": len(times), "text": self._encode(msg), "times": "<br/>".join(str(t) for t in times)}
|
||||||
)
|
)
|
||||||
|
|
||||||
def __init__(self, args):
|
|
||||||
super().__init__()
|
|
||||||
self.level = args["level"]
|
|
||||||
self.isOther = self.level == Action.OTHER
|
|
||||||
self.template = args["message"]
|
|
||||||
values = {}
|
|
||||||
for (_void, name, _void, _void) in string.Formatter().parse(self.template):
|
|
||||||
if name:
|
|
||||||
values[name] = None
|
|
||||||
self.values = values
|
|
||||||
|
|
||||||
def act(self, entry):
|
|
||||||
messages = Action._messages[self.level]
|
|
||||||
for (name, _void) in self.values.items():
|
|
||||||
self.values[name] = entry.get(name, None)
|
|
||||||
msg = self.template.format_map(self.values)
|
|
||||||
if self.isOther:
|
|
||||||
messages.append((entry["__REALTIME_TIMESTAMP"], msg))
|
|
||||||
elif msg in messages:
|
|
||||||
messages[msg].append(entry["__REALTIME_TIMESTAMP"])
|
|
||||||
else:
|
|
||||||
messages[msg] = [entry["__REALTIME_TIMESTAMP"]]
|
|
||||||
thisHour = datetime.datetime.today().hour
|
|
||||||
if thisHour < Action._hour:
|
|
||||||
self._sendReport()
|
|
||||||
Action._hour = thisHour
|
|
||||||
|
|
||||||
def _sendReport(self):
|
def _sendReport(self):
|
||||||
|
messages = [[], {}, {}]
|
||||||
|
with open(Action._storage) as journal:
|
||||||
|
for e in json.load(journal):
|
||||||
|
if e != {}:
|
||||||
|
(L, T, M) = (e["L"], datetime.fromtimestamp(e["T"]), e["M"])
|
||||||
|
if L == 0:
|
||||||
|
messages[0].append((T, M))
|
||||||
|
elif M in messages[L]:
|
||||||
|
messages[L][M].append(T)
|
||||||
|
else:
|
||||||
|
messages[L][M] = [T]
|
||||||
|
|
||||||
html = Action._htmDocStart + Action._htmHeadWarn
|
html = Action._htmDocStart + Action._htmHeadWarn
|
||||||
text = Action._txtDocStart + Action._txtHeadWarn
|
text = Action._txtDocStart + Action._txtHeadWarn
|
||||||
|
|
||||||
text += Action._txtTableDelim + Action._txtTableHeader
|
text += Action._txtTableDelim + Action._txtTableHeader
|
||||||
html += Action._htmTableStart
|
html += Action._htmTableStart
|
||||||
for (msg, times) in sorted(Action._messages[Action.WARN].items(), key = lambda i: i[0]):
|
for (msg, times) in sorted(messages[1].items(), key = lambda i: i[0]):
|
||||||
text += self._toAdoc(msg, times)
|
text += self._toAdoc(msg, times)
|
||||||
html += self._toHtml(msg, times)
|
html += self._toHtml(msg, times)
|
||||||
text += Action._txtTableDelim
|
text += Action._txtTableDelim
|
||||||
|
@ -85,7 +113,7 @@ class Action(base.Action):
|
||||||
|
|
||||||
text += Action._txtTableDelim + Action._txtTableHeader
|
text += Action._txtTableDelim + Action._txtTableHeader
|
||||||
html += Action._htmTableStart
|
html += Action._htmTableStart
|
||||||
for (msg, times) in sorted(Action._messages[Action.INFO].items(), key = lambda i: i[0]):
|
for (msg, times) in sorted(messages[2].items(), key = lambda i: i[0]):
|
||||||
text += self._toAdoc(msg, times)
|
text += self._toAdoc(msg, times)
|
||||||
html += self._toHtml(msg, times)
|
html += self._toHtml(msg, times)
|
||||||
text += Action._txtTableDelim
|
text += Action._txtTableDelim
|
||||||
|
@ -96,7 +124,7 @@ class Action(base.Action):
|
||||||
|
|
||||||
text += Action._txtPreDelim
|
text += Action._txtPreDelim
|
||||||
html += Action._htmPreStart
|
html += Action._htmPreStart
|
||||||
for (time, msg) in Action._messages[Action.OTHER]:
|
for (time, msg) in messages[0]:
|
||||||
m = '%s: %s\n' % (time, msg)
|
m = '%s: %s\n' % (time, msg)
|
||||||
text += m
|
text += m
|
||||||
html += self._encode(m)
|
html += self._encode(m)
|
||||||
|
@ -105,9 +133,3 @@ class Action(base.Action):
|
||||||
html += Action._htmDocStop
|
html += Action._htmDocStop
|
||||||
|
|
||||||
email.Mail(text, html).send()
|
email.Mail(text, html).send()
|
||||||
Action._messages = _resetMessages()
|
|
||||||
|
|
||||||
def _resetMessages():
|
|
||||||
return {Action.WARN: {}, Action.INFO: {}, Action.OTHER: []}
|
|
||||||
|
|
||||||
Action._messages = _resetMessages()
|
|
||||||
|
|
|
@ -101,10 +101,11 @@ def main():
|
||||||
"diff -U0 \"$0\"{,.test_ref} | grep -vE '^[-+@^]{2,3} |={5,}[0-9]+=='",
|
"diff -U0 \"$0\"{,.test_ref} | grep -vE '^[-+@^]{2,3} |={5,}[0-9]+=='",
|
||||||
f],
|
f],
|
||||||
check = True)
|
check = True)
|
||||||
assert false, "differences found in " + f
|
assert False, "differences found in " + f
|
||||||
except subprocess.CalledProcessError:
|
except subprocess.CalledProcessError:
|
||||||
pass # OK, no difference found
|
pass # OK, no difference found
|
||||||
_clean()
|
_clean()
|
||||||
|
os.remove('action_dailyReport.py.journal')
|
||||||
|
|
||||||
def entry(host, service, message, microsecond = None):
|
def entry(host, service, message, microsecond = None):
|
||||||
global _microsec
|
global _microsec
|
||||||
|
|
Loading…
Reference in New Issue