Commit 85499efb authored by Trevor Cappallo's avatar Trevor Cappallo
Browse files

add file manifest check utility

parent 51e6c607
#!/usr/bin/env python
"""Utility for checking whether files exist and conform to specifications."""
import glob
import os
import logging
import re
import OperationalLogger as OpLog
import OperationalMail as OpMail
OpLog.setup()
log = logging.getLogger(__name__)
class FileCheck(object):
"""Check files against specifications."""
PARAMS = ('size', 'files', 'count')
SUFFIX = {None: 1, 'k': 10**3, 'm': 10**6, 'g': 10**9, 'K': 10**3, 'M': 10**6, 'G': 10**9}
OPER = {
'=': lambda b: lambda a: a == b,
'==': lambda b: lambda a: a == b,
'>=': lambda b: lambda a: a >= b,
'<=': lambda b: lambda a: a <= b,
'>': lambda b: lambda a: a > b,
'<': lambda b: lambda a: a < b,
'has': lambda b: lambda a: b in a,
}
def __init__(self, filename, path=os.getcwd()):
if not self.load_rules(filename):
raise RuntimeError("no rules detected")
if self.check_rules(path):
log.error("failed to meet all file rules")
else:
log.info("all file rules passed")
def load_rules(self, filename):
try:
with open(filename, 'r') as f:
contents = f.readlines()
except EnvironmentError:
log.exception("could not load file: %s", filename)
else:
return self.parse_rules(contents)
def parse_rules(self, contents):
log.debug("parsing rules...")
self.rules = []
for entry in contents:
entry = entry.split('#')[0].replace('\t', ' ').strip()
if not entry:
continue
pattern = entry.split(' ')[0]
matched = False
for param in self.PARAMS:
match = re.search(r"\s%s\s?(%s)\s?(\d+)([kmg])?\s*(.+)?" % (param, '|'.join(self.OPER.keys())), entry, re.IGNORECASE)
if match:
matched = True
op = match.group(1)
num = int(match.group(2)) * self.SUFFIX[match.group(3)]
num_str = match.group(2) + (match.group(3).lower() if match.group(3) else '')
text = match.group(4)
self.rules += [Rule(
pattern=pattern, param=param, num=num, text=text,
cond_func=self.OPER[op](num),
cond_str="%s %s" % (op, num_str),
raw_str=entry,
)]
# if no explicit conditions, assume 1 or more matching files
if not matched:
self.rules += [Rule(pattern, raw_str=entry)]
return len(self.rules)
def check_rules(self, path):
path = os.path.abspath(path)
log.info("comparing rules to path: %s", path)
for rule in self.rules:
log.debug("checking rule: '%s'", rule.raw_str)
rule.okay = True
files = glob.glob(os.path.join(path, rule.pattern))
if rule.param == 'files':
if rule.cond_func(len(files)):
log.debug("'%s' passed file rule: %d %s", rule.pattern, len(files), rule.cond_str)
else:
log.info("'%s' FAILED file rule: %d %s", rule.pattern, len(files), rule.cond_str)
rule.okay = False
elif rule.param == 'size':
for f in files:
size = os.path.getsize(f)
if rule.cond_func(size):
log.debug("'%s' passed size condition: %s %s", f, size, rule.cond_str)
else:
log.info("'%s' FAILED size condition: %s %s", f, size, rule.cond_str)
rule.okay = False
elif rule.param == 'count':
for f in files:
contents = open(f, 'r').read()
try:
count = len(re.findall(rule.text, contents))
except Exception as err:
log.warn("error '%s' parsing '%s' as regex, treating as plaintext", err, rule.text)
count = contents.count(rule.text)
if rule.cond_func(count):
log.debug("'%s' passed count condition: %d %s '%s'", f, count, rule.cond_str, rule.text)
else:
log.info("'%s' FAILED count condition: %d %s '%s'", f, count, rule.cond_str, rule.text)
rule.okay = False
return not all([x.okay for x in self.rules])
class Rule(object):
def __init__(self, pattern, param='files', num=0, text=None,
cond_func=FileCheck.OPER['>'](0), cond_str="> 0", raw_str=''):
self.pattern = pattern
self.param = param
self.num = num
self.text = text
self.cond_func = cond_func
self.cond_str = cond_str
self.raw_str = raw_str
self.okay = None
if __name__ == '__main__':
FileCheck('test.fc')
# test filecheck manifest file
README # hash designates comment
## whitespace ignored
test.fc size < 10
test.f? size <= 10k
./test.* files = 1
lib/*.py sIzE== 1290 FiLeS>2
lib/*est*py files >2 size < 1GB
lib/FileCheck.py count=2 class
lib/FileCheck.py count<=1k def \S+\(self.*\):
lib/FileCheck.py count>0 rule.cond_func(
* # if no other parameters, rule passes if at least one file/dir matches
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment