#!/usr/bin/python2 -tt # # Processes dnscache log files as produced by multilog """ Expects input on stdin (fd 0) and writes output on stdout (fd 1) Writes state information to fd 5 and retrieves any such information on 4 NB: should run quickly; multilog may block while running this process Also, if your system time moves backwards, you'll get erroneous readings This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. A copy of the GNU General Public License is available from http://www.gnu.org/copyleft/gpl.html or write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. """ __COPYRIGHT__ = "Copyright (C) 2003, Michael T. Babcock" __LICENSE__ = "GPL Version 2" __AUTHOR__ = "Michael T. Babcock " __VERSION__ = "0.91" from pytai import tai class dnscachestats: '''Statistics information from 'stats' lines in dnscache log output''' # stats 19 10314 1 0 def __init__(self, copy = None): '''Initialize values based on log.c''' if copy: self.numqueries = copy.numqueries self.cache_motion = copy.cache_motion self.uactive = copy.uactive self.tactive = copy.tactive self.has_cache_hit_patch = copy.has_cache_hit_patch self.cache_hit = copy.cache_hit self.cache_miss = copy.cache_miss else: self.numqueries = 0 self.cache_motion = 0 self.uactive = 0 self.tactive = 0 # These are available if you're using the patch from # http://romana.now.ie/software/djbdns-cachestats.patch self.has_cache_hit_patch = False self.cache_hit = 0 self.cache_miss = 0 def __str__(self): '''Return ourselves in string format''' str = "Queries: %d, Motion: %d, UDP Active: %d, TCP Active: %d" % \ (self.numqueries, self.cache_motion, self.uactive, self.tactive) if self.has_cache_hit_patch: str += ", Cache hits: %d, misses: %d" % \ (self.cache_hit, self.cache_miss) return str def loglines(file): '''return timestamped log statistic data''' t, stats = tai(), dnscachestats() for line in file: taistamp, type, data = None, None, None try: (taistamp, type, data) = line.split(" ", 2) except ValueError: continue if not type == "stats": continue t.from_tai64n_ext(taistamp) data = data.split(" ") stats.numqueries = int(data[0]) stats.cache_motion = int(data[1]) stats.uactive = int(data[2]) stats.tactive = int(data[3]) if len(data) > 4: stats.has_cache_hit_patch = True stats.cache_hit = int(data[4]) stats.cache_miss = int(data[5]) yield (t, stats) opt_verbose = False opt_stats = False def logprocess(data, inittimedelta, initmotion): '''Process the data returned by the generator''' totalhits, totalqueries = 0, 0 firsttime = None firstmotion, motionmod = 0L, 0L laststat, lasttime, lastmotion = None, None, 0L for (time, stat) in data: if opt_stats: print "%-30s %s" % (str(time), str(stat)) if lasttime: # Process data for statistics timedelta = float(time) - float(lasttime) if stat.cache_motion < lastmotion: if opt_verbose: print "We wrapped around; restart probably" # Could check queries count to verify motionmod += lastmotion if opt_verbose: print "Time delta from %s to %s: %.3fs" % \ (str(lasttime), str(time), timedelta) if stat.has_cache_hit_patch: totalhits += stat.cache_hit totalqueries += (stat.cache_hit + stat.cache_miss) laststat = dnscachestats(stat) lasttime = tai(time) lastmotion = stat.cache_motion if not firsttime: firsttime = tai(time) firstmotion = stat.cache_motion if not lasttime: return 0.0, 0 timedelta = float(lasttime) - float(firsttime) if timedelta == 0.0: return 0.0, 0 motion = stat.cache_motion - firstmotion + motionmod print "Overall time: %.2f seconds" % timedelta #print "Queries: %d" % queries print "Cache motion: %d bytes" % motion dailymotion = motion / (timedelta / 86400.0) print "Extrapolated motion per day: %ld bytes" % long(dailymotion) if inittimedelta and initmotion: motion += initmotion timedelta += inittimedelta print "Including previous data:" print " Overall time: %.2f seconds" % timedelta print " Cache motion: %d bytes" % motion dailymotion = motion / (timedelta / 86400.0) print " Extrapolated motion per day: %ld bytes" % long(dailymotion) print "TTL-average 3 day motion: %ld bytes" % long(dailymotion * 3) print "Full-week motion: %ld bytes" % long(dailymotion * 7) if totalhits and totalqueries: print "Current hit-rate: %.1f%%" % (totalhits * 100.0 / totalqueries) return timedelta, motion def state_save(state_data): '''Save current information''' import pickle from os import fdopen try: print "Saving current state (failure is acceptable)" f = fdopen(5, 'w') pickle.dump(state_data, f) except OSError, e: print "OSError: ", str(e) except pickle.PickleError, e: print "PickleError: ", str(e) def state_load(): '''Load previous run-time information''' state_data = None import pickle from os import fdopen try: print "Loading previous state (failure is acceptable)" f = fdopen(4, 'r') state_data = pickle.load(f) except OSError, e: print "OSError: %s" % str(e) except pickle.PickleError, e: print "Error in state information: %s" % str(e) except EOFError, e: print "No state information available: %s" % str(e) return state_data def usage(): print """Usage: dnscacheproc.py [-h] [-s] [-v] -h This help message -s Include line-by-line statistics -v Be verbose""" if __name__ == "__main__": from sys import stdin,argv,exit # Handle command-line arguments for arg in argv[1:]: if arg == "-v": opt_verbose = True if arg == "-s": opt_stats = True if arg == "-h": usage() exit(0) timedelta, motion = None, None try: timedelta, motion = state_load() except (ValueError, TypeError): print "No (valid) state information, starting afresh" timedelta, motion = logprocess(loglines(stdin), timedelta, motion) state_save((timedelta, motion))