"""Snapfilter
Copyright (c) 2009 Scott Ludwig

Snapfilter is a filtering tool to determine which zfs snapshots to retain, and
which to remove, according to a user specified retention policy.

The retention policy is specified with command line arguments, and can
by specified with yearly, quarterly, monthly, weekly, and daily periods.
Each setting, if not specified on the command line, has a default:

2 Yearly backups will be kept by default
4 Quarterly backups will be kept by default
6 Monthly backups will be kept by default
12 Weekly backups will be kept by default
28 Daily backups will be kept by default

The snapshot list is read from stdin. Items in the list are assumed to
use the '%Y.%m.%d-%H.%M.%S' strptime format at the end of each line. For
example:

tank/data@daily-2008.12.14-01.02.03
tank/data@daily-2007.01.22-01.02.03
tank/data@daily-2006.04.19-01.02.03

Command Line Arguments:

snapfilter [-t] [-s] [-v] [-y #] [-q #] [-m #] [-w #] [-d #]

-t      Run unit tests, and exit
-s      Output snapshot list sample, and exit 
-v      Verbose output
-y #    number of yearly backups to keep
-q #    number of quarterly backups to keep
-m #    number of monthly backups to keep
-w #    number of weekly backups to keep
-d #    number of daily backups to keep

Examples:

See sample verbose output, with a 60 day, 52 week, and 5 year retention policy.
(snapfilter -s just provides sample input):

./snapfilter -s | ./snapfilter -v -d 60 -w 52 -y 5 | less

See just the removes from the sample input, with the default retention policy:

./snapfilter -s | ./snapfilter

Example script to be run once a day via cron job. This script makes a snapshot, then deletes snapshots that are no longer in the default retention policy.

zfs snapshot tank/data@daily-`date +%Y.%m.%d-%H.%M.%S`
for snapshot in $(zfs list -r -t snapshot -H -o name tank/data | grep "tank/data@daily" | python snapfilter.py); do zfs destroy $snapshot; done

History:

v 0.1 04/21/2009 scottlu
  - first version

License:

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import re
import md5
import sys
import time
import base64
import random
import datetime
import cStringIO

snap_re = re.compile(r'^(?P<label>.*)-(?P<time>....\...\...\-..\...\...)$')
date_format = '%Y.%m.%d-%H.%M.%S'

class Snap:
    def __init__(self, name):
        m = snap_re.match(name)
        self.s = time.mktime(time.strptime(m.group('time'), date_format))
        self.label = m.group('label')
        self.name = name

    def __repr__(self):
        return self.name

class Period:
    def __init__(self, keep, snaps):
        self.keep = keep
        map = {}
        for snap in snaps:
            key = self.key(snap)
            if map.has_key(key):
                if map[key].s < snap.s:
                    continue
            map[key] = snap
        keys = map.keys()
        keys.sort(reverse=True)
        self.snapmap = {}
        for key in keys[:keep]:
            snap = map[key]
            self.snapmap[snap] = snap

    def has(self, snap):
        return self.snapmap.has_key(snap)

class Yearly(Period):
    def key(self, snap):
        t = time.localtime(snap.s)
        return time.mktime((t.tm_year, 0, 0, 0, 0, 0, 0, 0, 0))

class Quarterly(Period):
    def key(self, snap):
        t = time.localtime(snap.s)
        return time.mktime((t.tm_year, (t.tm_mon - 1) / 3 * 3 + 1, 0, 0, 0, 0, 0, 0, 0))

class Monthly(Period):
    def key(self, snap):
        t = time.localtime(snap.s)
        return time.mktime((t.tm_year, t.tm_mon, 0, 0, 0, 0, 0, 0, 0))

class Weekly(Period):
    def key(self, snap):
        t = time.localtime(snap.s)
        return time.mktime((t.tm_year, t.tm_mon, t.tm_mday - t.tm_wday, 0, 0, 0, 0, 0, 0))

class Daily(Period):
    def key(self, snap):
        t = time.localtime(snap.s)
        return time.mktime((t.tm_year, t.tm_mon, t.tm_mday, 0, 0, 0, 0, 0, 0))

class Filter:
    def __init__(self, snaps, argv, defaults):
        if argv.__class__ == ''.__class__:
            argv = argv.split(' ')
        self.argv = argv
        if defaults.__class__ == ''.__class__:
            defaults = defaults.split(' ')
        self.defaults = defaults
        
        # Read in snaps but allow them to be passed, for UnitTest speed
        if snaps.__class__ == [].__class__:
            self.snaps = snaps
        else:
            self.snaps = []
            f = cStringIO.StringIO(snaps)
            for name in f.readlines():
                self.snaps.append(Snap(name.rstrip()))
            f.close()
            self.snaps.sort(cmp=rcmp)

        # Distribute snaps to period intervals
        self.periods = []
        self.periods.append(Yearly(self.int_arg('-y'), self.snaps))
        self.periods.append(Quarterly(self.int_arg('-q'), self.snaps))
        self.periods.append(Monthly(self.int_arg('-m'), self.snaps))
        self.periods.append(Weekly(self.int_arg('-w'), self.snaps))
        self.periods.append(Daily(self.int_arg('-d'), self.snaps))

    def int_arg(self, arg):
        try:
            index = self.argv.index(arg)
            if index + 1 < len(self.argv):
                return int(self.argv[index + 1])
        except:
            index = self.defaults.index(arg)
            if index + 1 < len(self.defaults):
                return int(self.defaults[index + 1])
        return 0

    def remove_str(self):
        remove = []
        for snap in self.snaps:
            keep = False
            for period in self.periods:
                if period.has(snap):
                    keep = True
                    break
            if not keep:
                remove.append('%s' % snap)
        return '\n'.join(remove)

    def report_str(self):
        r = []
        for period in self.periods:
            r.append('%s: keep %d' % (period.__class__.__name__, period.keep))
        for snap in self.snaps:
            a = []
            for period in self.periods:
                if period.has(snap):
                    a.append(period.__class__.__name__)
            if len(a) == 0:
                r.append('%s REMOVE' % snap)
            else:
                r.append('%s %s' % (snap, ', '.join(a)))
        return '\n'.join(r)

def rcmp(a, b):
    if b.s > a.s:
        return 1
    if b.s < a.s:
        return -1
    return 0

def sample(count=365, spread=1500, seed=7919):
    # Produces a sample for testing purposes
    r = random.Random(seed)
    s = datetime.date(2009, 4, 18)
    a = []
    for i in xrange(count):
        d = s - datetime.timedelta(r.randint(0, spread))
        a.append('tank/data-backup@daily-%04d.%02d.%02d-01.02.03' % (d.year, d.month, d.day))
    return '\n'.join(a)

def index_arg(arg):
    try:
        return sys.argv.index(arg)
    except:
        return None

def main():
    if index_arg('-t'):
        tests = UnitTests()
        sys.exit(tests.do())

    if index_arg('-s'):
        print sample(seed=random.Random().randint(100, 1000))
        sys.exit()

    if sys.stdin.isatty():
        print __doc__
        sys.exit()

    filter = Filter(sys.stdin.read(), sys.argv, '-y 2 -q 4 -m 6 -w 12 -d 28')

    if index_arg('-v'):
        print filter.report_str()
        sys.exit()

    print filter.remove_str()
    sys.exit()

class UnitTests:
    def __init__(self):
        self.snaps = []

    def do(self):
        if self.test_daily() != 0:
            return 1
        if self.test_weekly() != 0:
            return 1
        if self.test_monthly() != 0:
            return 1
        if self.test_quarterly() != 0:
            return 1
        if self.test_yearly() != 0:
            return 1
        if self.test_combined() != 0:
            return 1
        if self.test_remove() != 0:
            return 1
        return 0

    def test_remove(self):
        a = (
        '616335a740fdd2a33d47711acf03db87','332c2d47e34a7fd99c83ccbcb13ab3aa',
        '290606fc269f624d33084d796a7861d5','b5d5500c96b58f4064b200068e3dc685',
        'e4b4ca05b9396431980b0420442e430a','03878cc72bdf8ed08b3f53298d1b5831',
        '9010f74f7b3d3fcd44615c540ce97794','78411aa86a0cc6565461caa09f3d4e71',
        'bcfcc0c40db63b0290188f3290ed4827','aec93fac1e7694dd78ed94786d7d531d',
        '4a85251d2b8543f2db41d984ff99585b','ef4684220c4dc651dfc0a127381e4e7f',
        '6f1701c72a511e3ad1495a7607ea5adf','5b324f5b95d4d870cf60b0ad15f7cb9d',
        'c9b08ec6293e3038701641e4a12601ae','026a338012be682e74207f983a614098',
        'a35064d08e53d236d8bab22be9abc979','8ec9cb3c6e76663ca0502b28cdc3dbb7',
        '14900fefea9f52e2f1b93936d6c47d7a','632c0c199c3ae72a434ecdcbee05515e',
        '5665b1140d2b2fc3dca8c3849bca14bd','20e6bdf8e659ca7ff6bb67c7d548059f',
        '871fd70a1de9336141d7aec6d9b8d727','32a090492ee96aa06d9eb7968ff0fb2f',
        '66af27379d87cdd30d957ca0c5177fac','76a3a2742b68086b6912aef30630ee3e',
        '1e5296e610a81fcc144d99e76b51dbc0','15c3aab70d85f68641364b1360dc5869',
        '15a5552787656519d2555150a231eb2f','7b048aa78d0c8696ddbfd6a0d5eb934d',
        '35dcce8c2364ea0fa734965fc5a0258b','374d68d71672f5469fd130ccf174b0f6',
        '6f80cb79b0f74726766c49dd817cd7e0','2e0abc8cfec04d9cdeec94cffc47dcf6',
        '54089d76250fdc11f7bc7211b1f6c0d2','2ca71eeb55ec470bdea7e2ca0b47e882',
        '9462bbf7fe3d6efbe5c05a565a89a75e','fd66ffd85e59c37fcc016ad1f4530ffd',
        '75c22dd1eb07f4ce4c887a1b7ba4f113','6eda905bb3a2a867a9ce1c53f53ca3cc',
        '9bac582fb9d91c659a12a0742fff09b6','0ba351138ae1759cfe0133ebd38bd30e',
        '26dd59d5cd0c16a65dad09e84a0465db','442b5b0a57f81fceb83e8312c35bb95e',
        '6d7e20f1596a4dba32b6882dcccf5ae8','3f4f4ffae79cf1772380377f712645b5',
        '8590bf800390595f4af7f6b666741721','7264651a9d04fd012e14ed2f342c4587',
        '17e6987e79fbfc4cad13ad75569b263b','8731bdeb250a77ba236e1d6856bd8f0f',
        )
        config = {}
        config['answers'] = a
        config['intervals'] = 1000
        config['intervalstep'] = 20
        config['spreadmult'] = 20
        config['seed'] = 1031
        config['argv'] = '-empty %d'
        config['defaults'] = '-y 15 -q 30 -m 48 -w 156 -d 365'
        sys.stdout.write('remove test...')
        return self.run_removetest(config)

    def test_combined(self):
        a = (
        '235a4abc5bf0ab7727ecabac26c735c2','9a58af425c1c6824a9f4099ca053b0ab',
        '85b99f591aec6672f5e1102411455210','9449583202c9d44d33f3537016fc0353',
        'a1a5fee7c7ba0221db28bd7bafe4c988','bf9c666cb7ddd228fe950ac34a25d9bf',
        '5e64c1803b953d38a5d015ca586b0c63','be2cdeacdfd590f44c36339b6e00f0a1',
        '338e9b155097e283e5216b338a5e14fe','ead753bad9216e73079a10b7f0b34ee3',
        'def5ddad572125bb8122e829e621cc95','2750cf98f8a9ddf14f3d7f7603563a0d',
        '6bb17e4dfe0bb49bb9be7d52eed84367','b9a4806f0d0b2881c7c849ffd4ac8f90',
        'a4b1fd865dae6711b8e81041afee853b','528f858bd05a8e62f3653a734699d86b',
        '1727f580ae2a5692bfde3bf08cc9667b','4a9c0b10aec62f5e92b58d0ae479144c',
        'a05835b715e333a1432c14581c79e311','96fa81947bb7b4bd044dbced12325fb2',
        '3021bb4b7e66996b1e598196598e279d','53cec9f5fc10a0ef41380e536fe3d34b',
        '79cd137bc792491665e3f218183328a4','a4489b50aa8c723b1786a166ce1b10d2',
        '1850e844ba1697e80a05ec4c04ddb97d','c2dd44a842715665996b5626f3f35254',
        'c066e1d17cda068911f4e0d00299a4f8','d01df2cfdb25a21e04bf68afbb44c942',
        '5a1b6f6bb717ef0227b2818ad8f99940','ff03906708d10ec2859a7cf670019ee2',
        '000c962b9f302eac4b60c0e4a12766f0','b20230c8177996abbc04997b7c2aea0c',
        '02ac68f2303ecdf2aef65608e3c63b91','5dc294c0770c0284f2ed46c72933ccdf',
        'fa31a049ab3bf5c0c89707d9eea9ba03','48823f50bbaa9b2fc2a7a46360029b00',
        '0ace0af87a95e31b9de97675883a524d','6f9b87e58cddcfd97c2f19d6a0f0cd65',
        '6bd44f1f4476a4332f701306f644f8da','dbf70cd371c7f529cd095bb508cd65aa',
        'bb104cd4dce2c7a0f5f6a6047b365beb','2d465677d6d673a90fd9a28ec5903b5c',
        '92df8a920aa6fb9480c3f2854983d18f','457a90b3ee023def0c87d92435f9ceb6',
        '30ff2aa3703b1ca268ba36ec74b50038','ccca5f1631def8a7f9b0768bde3dbab7',
        '02c5b10e93486799fd330d1d96b094b1','5cbaf1b8d0505832a2aec65ecd92e404',
        '1e30dd904612dba76c529822e7a46765','43407c666b5f3053a90c2a5cb6789367',
        )
        config = {}
        config['answers'] = a
        config['intervals'] = 1000
        config['intervalstep'] = 20
        config['spreadmult'] = 20
        config['seed'] = 7727
        config['argv'] = '-empty %d'
        config['defaults'] = '-y 10 -q 20 -m 36 -w 104 -d 180'
        sys.stdout.write('combined test.')
        return self.run_reporttest(config)

    def test_yearly(self):
        a = (
        '3c3562c377486af4d8d2b249e2294668','1a7ac6b52b5cc063eb70b004d31bc243',
        '964921eee3ff2455f2b9f6183cedb1fd','091ffac596638e7edbd5ee05a796d49d',
        'bae09456bd50e7438ff0c1a18479959a','d31561848d94e951de0c4dea3eda498f',
        '9c21f47eb6bd74efdda593a8f8cefcdf','d53dde603230671d3fa73edb30f01976',
        '861a0ec9c15726da689a9a172fc87a12','accfee39deed03c93b7c93504b4a0d1f',
        '27e128c8f3c19d95b70aa2ec868b32be','024c2000111765cb643f09f1d36220a5',
        '5b77335391f041f2c824c64f56239646','94965e3d441dd305d2eeedd6ecadaf26',
        'c3e63b25da93ae3021686ea6030ed038','213289bee75680b64c93c45d87232f68',
        '634c0d794f45308faae633a5f772d6bf','dc277af28366c08bbcefd31fe71c81c4',
        'fc4e5e3c1d06165b73284a4478810c6c','56facc81432969af2b5c37991bf923b6',
        '94925d86d1fab92ad71b6da37bc4ca45','b8fc80cb16bdb9fe0b327c47ff5ad5b9',
        '5b4a5f9f00a48bd292d3f9baee9bf634','7e011594fc98d408e791a39cfb563029',
        'adeb8665cba2220469164c7055066f16','315a765ae48ce4b418d7716e06e5e2f0',
        '12cd160aa6d51f6a432a5d80ed252b4d','9c2b416d094282da8a5fb9dcf230f6cd',
        '397836b539d83b77713c22995c610934','d4351a4a8ed64e0a28dc81e5f12a2dfd',
        '0037d7bcf23599dbb560903a5a356531','bfc6ca440e29783bf92946df60a87a01',
        'ded35d787b908a27b6e20877b2480673','bc153f187ccfeb33417e8365695c128e',
        '0b98832ebbf974f7a5e14d95fa64b44c','7e4877d272727dc0ead539da4d4c0566',
        'd881e20056fde9f1caeff502bb1576ec','e907dac960abdb31dacebaebdb3898d8',
        'c2b941cb7b960be131fadeabab38b62b','7d66559e63268adc790ed9fbe25b5f6f',
        'f51a937d41bbbb96a5a5b4c05b0cb677','4608a7b9a3a3f4d73170b92929677001',
        '352cfd20954ce3a42cb7c2f77ebb551b','d72bd75d647a2df1582babcb28d422ef',
        '9841a9940c10b82fe9e82543e50fb873','1d4a8de8516e24cb30d9bf7bd2bd3f18',
        'ee542d56768c71c2103275345d41dae1','723db4bc815d13115fa3dbebfbe48708',
        '32b4623b133785ae45c14aa84d1b6a2c','87ac1edd607b0319703305a8190fd304',
        )
        config = {}
        config['answers'] = a
        config['intervals'] = 100
        config['intervalstep'] = 2
        config['spreadmult'] = 1
        config['seed'] = 1319
        config['argv'] = '-y %d'
        config['defaults'] = '-y 0 -q 0 -m 0 -w 0 -d 0'
        sys.stdout.write('yearly test...')
        return self.run_reporttest(config)

    def test_quarterly(self):
        a = (
        '58da7dadf64b555ff9670abee2bd6be1','a5304b6a62e202161ffa74a05bb1e909',
        'c08c99a56688b156e4e427e17d574954','bee529e598b35c1068e7ecf48fb1f0d2',
        '5260909571d0ac6f934a51bca09dcdd8','b6152deca2dc8f4759d4c42c7e50f1e7',
        '47e0d8d9c92d6c508322388a0e59fa55','1b81de0c7305bbaeed09b397a2fb70d0',
        'd4ce5e3797ec32485d942a8054938cae','91c8419463de4c512f8bfe4825a67a4d',
        '559835c1bbe286172c185ac5dc0ea526','f51b7b3b79b84745eecd1cbdd38c3983',
        '70cd044e4be9a99ee3c94fa41c65b75a','fd831039a79f253b5ffff8cc235bdcdd',
        '995827318a8329a38010143ce8a0fbb6','199e9c12210a9656eafa76d487faaf79',
        '50020c81d3a59c4a8b550df267d4abdb','d1de5e57c4e575103c2b3c8c52d06eec',
        'd07c239ec0c1186ae7a1673ffe205514','26222c73d2199c06d4681c5076f6dc7d',
        '0ba21b6334ce2cb6b79ac1adcdbb5b2d','6f184abed5305e47b35b80280452a7c5',
        '3724bb7f9ee9eb08b918226bbcb1c159','951b058dbb5606ff2f3a3cd746180d11',
        'e71d781e1088eefbbc1c2d1c3a313dc0','aea51afb9e6cffc63a8716b5e419207c',
        '1cef029e72a289f023360865e88bbfc7','63feb8e2e17cbfc1d4164fa2d89bb9f9',
        'f025735de67c1dbdc7dcb22cd2014b45','c03a5fe8456113daab121fe7077513c2',
        'efb68a2e2873d7054e5a4c2c938cf22b','4f91c51f3016681f037fed2ac096e914',
        '3e3fe415ebcb13f7d0da72a394396bb5','e4a235f0299955335973871875334d1b',
        '5a9b281fb6c814cc00d4e223bc1d9643','c8900495200b62dcf2bf6ae5e0fddef4',
        '4a3b176401f4eb2010ac9e5d2f780e0e','9395ec6ced9cb1bcf9942a04ef719317',
        'a4485cd6c626fb882e63121280da6ad3','cde6e5cc11babe78cf8a859a48ec8c38',
        '7d403506c8b1efed3ebc86570e6419fc','b60f76c7b6cc0c6335df6e0372d28855',
        '4d475ecefb479a9736547afd65c7999e','9a4226664f7e0e5b7ad7994ee71d45c9',
        'e49b94a4c83443be4af90ace1f6d957c','4e779fde84dd278c6c27956d0cac756b',
        '54d796c3bfb01e868e59ad7e9ead3189','c079cc5a0899886a6d3a9be9388142bf',
        'bfd94045d2dbfd4e79bad3179a1d8b0c','94e61c0b78bfecd06901e994b28d0820',
        )
        config = {}
        config['answers'] = a
        config['intervals'] = 100
        config['intervalstep'] = 2
        config['spreadmult'] = 90
        config['seed'] = 5023
        config['argv'] = '-q %d'
        config['defaults'] = '-y 0 -q 0 -m 0 -w 0 -d 0'
        sys.stdout.write('quarterly test')
        return self.run_reporttest(config)

    def test_monthly(self):
        a = (
        '37771387b5326c842730c2d90be25705','54c8f9168386578ef89871bfec80772b',
        'e5cc549a6f64aeefe75bc689d6d21427','c00cde1a0e1f9328bd6d717254503a07',
        '95508f5ce698065b2147515694641cb9','9cc1d53f99e3bfc9237aab9ecfcfb81f',
        '2714b29057dac71cf4dccad771dc1bf0','b1e9477e4645dbcc29d93289935cb985',
        '0ba517b7d704ff6f006c2ca8a9ae13a0','2c61b743effda63f16476ef2de02dc8b',
        'b1b8e3565d56619b89eb45b3d89d0a85','bb8829a25c6c7788da9539b74f4c8bb2',
        '2c2c750588c4677fe8b58ffa3fb3e580','832a9f389a7f6d843f0d5b007864ba99',
        '4bedf6aa9912d033524e6102c62dc0c5','22a22fa3371e4459230b1c196314bc42',
        '5e50130046b9980e272af59bd596bbc0','81f9639e152d9d666411eef85edc6f06',
        'eee9001ce606d9a794d2902cdfb26164','5a8868162daf262f69d769946708af45',
        'f5f88dbdfbd4129159501b6a9256e413','2c143a35596aa27350d7c7c3dd092a21',
        '62022466941201988614c8126fe10208','cd73af9cc35727266a1656a15f59d8df',
        '837d4e677cdd77646460326a4f4233dc','026e256b0d0dd78751744162350212dd',
        '149c03fcd1ce03ed53291558e5221d59','e17851547c5231726f888db2076baee0',
        '08ff28713f2d39d60bd76e4549e7cd78','c1abe9bea7d87c6cdda0dd70283b56f0',
        'abfd09b874c6230c990de33d2b8d9437','c9107cdceaa26e0428105fa086440479',
        'a0d06625f0053584479f571199d87662','61847152a027a44d4437d0a269dda3ea',
        '2245449c02ac5cf16a65efb177fc483d','75173454cb1afb5e313724bfbbea6da6',
        '9bdb42da302a5729f7613859dd83f31d','5a9dd178f22808da361fd93d9100c107',
        '3fdfb0e17705da4de82b6a7c788fd2cc','2b837e4d3b44fa628eef3d2edf2636f0',
        '023834fd68406f0661691d795f13607b','63d248a9439936241a195aa5e45058bc',
        '091260a51bb954f88bba94315b12416f','dff93842cb006b1042176d7d17b70a9c',
        '666504956d86ce8ccb6fbbd6a187a776','528c49334b6f7e289870ca7ab639c0fe',
        '976fe7f66c802edea67f446f6ff60933','80237587d01926a999841d5ce875d749',
        '31c9ccc89809f116beef1477840416d0','fd3a1b622751d478083a34d50a0d0f06',
        )
        config = {}
        config['answers'] = a
        config['intervals'] = 500
        config['intervalstep'] = 10
        config['spreadmult'] = 30
        config['seed'] = 7109
        config['argv'] = '-m %d'
        config['defaults'] = '-y 0 -q 0 -m 0 -w 0 -d 0'
        sys.stdout.write('monthly test..')
        return self.run_reporttest(config)

    def test_weekly(self):
        a = (
        'c8b31b0b1f862dcb24cd4c879dd191af','27273213e258492cae6fc27b5999b18a',
        '75d202d74d8f980bfa0aa2c53e014e57','2c02d60a36a94a1d0af4cd5fd00ce660',
        'c32c5b79f5e2237d293c72c6a7ee9e1c','726906e8398a260738458d6a72dc00f3',
        '41d5f80ec85fecc13c690cca0861b77e','980722c24f87f3e28bee59d0875572fc',
        'a5d348a14d9bc14eba961dd189183c51','74880a44a1e4a19d0197fef122dc0ce5',
        '501d85defd86728f959759af6e73eb94','e8474d3da21aafb06f0954fb20b9f78c',
        'bc3795c9836518857a0a58bfd2142a37','082095bb3392a9f67afff77ad3e981a2',
        '8296c350b50339a4f19e109d475f7d51','bf3388e6e48e1f1aace642241f60a5ec',
        '6b7ca40344dbd955dc45405df5ae01b6','9af11ead39730238b83c6f157ef7eb8f',
        '941ea64a27a374143ca549a4c462203a','6e626bd32a92645e0b3c9c3d74994161',
        '5fec239a549e0484e995360779570a35','671f8cbe884bc7a36c474c1b7c93da5b',
        'd511b36e42156349ed287c6bb9d57963','c0119a1b4a5f7a6a8c34be9706f30893',
        '0b9710dbc770e549ce0a8831d59e4e73','2136aeede926e30d479c528d023af282',
        '052053ed8e7fa59c1dc052db05e9efc1','412dc947e0da49de40f25ce80a7364ba',
        '82dd74ff74dfcb032006ca7f1211ed51','77a8ba761a643313f2d33f47d3229076',
        '883340a25ae1b098b263cf6dfbf1129f','d6e6db855f283693e774b3e68e227dd0',
        '8ad4dafc3f1d72705b48507ea985a60e','9a1219e3327479787b2df54edf04b893',
        '75cfe98bdd6b42687e96f7fe2393b94a','785cdc3c56eb61f94593ecff5abac9c6',
        '1beb8187a36e8fbd31f768144e56c137','ae85932a45cfa960aefcde31c5b4d217',
        '31d481969fb1d24feac5171f043c0f2f','72417c25cd989e10926cce543ec78b74',
        'a41dabbf88bcb17510d11073ecf39f9f','9e831ffe82528b2412a18f580325ff7c',
        '31bbd8fa4474ad4177b58fc6c84dbd99','13c5280337c0b91e9c70b9e3fea94e22',
        '34bbc225162aa3c7e57b9828a4398d85','394ee8dfd8a12450bc15fc6cec6f9c54',
        '1343e8bcc5b5c684448fdbd33a1ae669','11fc2ca471eeab0874b81ef2bd2f2492',
        'be2443e57f58f7f97e5b46836287b78a','cbfe9b6822cb9fa9991800bbcb638108',
        )
        config = {}
        config['answers'] = a
        config['intervals'] = 500
        config['intervalstep'] = 10
        config['spreadmult'] = 7
        config['seed'] = 6427
        config['argv'] = '-w %d'
        config['defaults'] = '-y 0 -q 0 -m 0 -w 0 -d 0'
        sys.stdout.write('weekly test...')
        return self.run_reporttest(config)

    def test_daily(self):
        a = (
        '9a0bf2399b7f76cfcb370612998c5bb0','ca64e7ba841be76508020c8f62869203',
        'c0d60f25cdd6d0e7d2514ce35071f7b7','be093bacc834faab2ba308ef200494f4',
        '9272cb7a14335896b6038e8ffc50008e','372172b8c7fe9bd769215e7b30ba3c9b',
        '4d0884395e827091f87638efa28819f3','943bb5584259584b814a03a08edaac32',
        '449988af59ef562060bcc462b64eabcb','71e3be3bf25b77c25de6995f82683797',
        '6ba008c85e10da14d6d6a08284a8c713','18c44049460c7cfb19f70abbabcee090',
        '2b792b17f80d5853ad2e1930b27507ff','e7c1e3d1f62ed939cd4467b8bb85ba01',
        '0c4f65bd06b9408567d29c5392556579','4417595e8a704e676916d4a338ac8801',
        '85fc6d62eb1e313e3697240b15ead6a4','1c657e9997e344382280f16509b86730',
        '454fce4be2ee4bd3e74a88a3e238de21','9d4bc566a540cc63785ba7b8e371dac9',
        '0f14f0a34cf88b8b6c225c31dcee88a2','6949b09bd4196d0616e082aa7519e34e',
        '95ead5d05cfb44b2382167f3b954063d','77570924367a7e50d2bab55c7aad8e5e',
        'f19cc733b13f26cbdcdb4eabc2aef5cd','a6f53add3edc0e5f9b070b9272d90dfd',
        '38af3dbf9c8ed721a0b9573d9a11d161','95c570b3974cabc7a013cb2b47f5b1b6',
        '1da09e9477332f6df4bd88eb7bd00fb5','c9c0a000e0ea19377cc5254c30d09260',
        '031f3c2be7e213b2cdbf9f09f5771065','e0ffe934f4c9b08e1a99e8047b97ac72',
        'cd656b5e9df1e48986bd61ce7bfd6493','d6c8c9bb549fdbc4aee0abc1f6707a57',
        '31d8d97a23d7cbdfd720bc11c8a45f09','6552879f8f43de479621c9ff9d5c396c',
        '3618cb64b33a29dceb17088e17c8866f','40d69ab4ab2b3e47c349f58bc3b30cd6',
        '4acf89a17dd1e062b6a8cb439008650f','13118cdacc71ec5565b9762784ca7836',
        'dc045bf38bb07da936c5e62c67ffe893','0d99101f7d4cb938f0534e678a1817c2',
        'd216223eccac1cbe22626381d3602b64','05ce31dcd76f55a9fb2731fd89c52a38',
        'f789cde9b757802b681ba216494413b4','98495184c6db231219f836dab672a6dd',
        'e6e74f1905ddc2f666a97a6171d24787','90d75aecf31279064bf129fc4316d201',
        'bd4b6d5abc86d5d397c9820a6405d372','7123fa98a24bce9f9b96c85c86677389',
        )
        config = {}
        config['answers'] = a
        config['intervals'] = 500
        config['intervalstep'] = 10
        config['spreadmult'] = 3
        config['seed'] = 2789
        config['argv'] = '-d %d'
        config['defaults'] = '-y 0 -q 0 -m 0 -w 0 -d 0'
        sys.stdout.write('daily test....')
        return self.run_reporttest(config)

    def setsnaps(self, count, spread, seed):
        if index_arg('-v'):
            print 'snaps count=%d, spread=%d, seed=%d' % (count, spread, seed)
        self.snaps = []
        f = cStringIO.StringIO(sample(count, spread, seed))
        for name in f.readlines():
            self.snaps.append(Snap(name.rstrip()))
        self.snaps.sort(cmp=rcmp)

    def run_reporttest(self, c):
        # tests snap counts and distribution correctness
        r = random.Random(c['seed'])
        index = 0
        sums = []
        for i in xrange(0, c['intervals'], c['intervalstep']):
            count = r.randint(i + 100, i + 200 + r.randint(0, i))
            spread = count * c['spreadmult']
            self.setsnaps(count, spread, r.randint(2000, 9000))
            sys.stdout.write('.')
            sys.stdout.flush()
            filter = Filter(self.snaps, c['argv'] % i, c['defaults'])
            report = '%s\n' % filter.report_str()
            if index_arg('-v'):
                print report
            rh = md5.md5(report).hexdigest()
            sums.append(rh)
            if c['answers'] and rh != c['answers'][index]:
                print '+++ unit test error!'
                print '+++ report:'
                print report
                print '+++ report hash:'
                print rh
                print '+++ correct hash:'
                print c['answers'][index]
                assert False, "Unit Test Error!"
                return 1
            index = index + 1
        sys.stdout.write(' PASSED\n')
        sys.stdout.flush()
        if index_arg('-v'):
            for i in xrange(0, len(sums), 2):
                print "'%s','%s'," % (sums[i], sums[i+1])
        return 0

    def run_removetest(self, c):
        # tests snap counts and distribution correctness
        r = random.Random(c['seed'])
        index = 0
        sums = []
        for i in xrange(0, c['intervals'], c['intervalstep']):
            count = r.randint(i + 100, i + 200 + r.randint(0, i))
            spread = count * c['spreadmult']
            self.setsnaps(count, spread, r.randint(2000, 9000))
            sys.stdout.write('.')
            sys.stdout.flush()
            filter = Filter(self.snaps, c['argv'] % i, c['defaults'])
            removes = '%s\n' % filter.remove_str()
            if index_arg('-v'):
                print removes
            rh = md5.md5(removes).hexdigest()
            sums.append(rh)
            if c['answers'] and rh != c['answers'][index]:
                print '+++ unit test error!'
                print '+++ removes:'
                print removes
                print '+++ report hash:'
                print rh
                print '+++ correct hash:'
                print c['answers'][index]
                assert False, "Unit Test Error!"
                return 1
            index = index + 1
        sys.stdout.write(' PASSED\n')
        sys.stdout.flush()
        if index_arg('-v'):
            for i in xrange(0, len(sums), 2):
                print "'%s','%s'," % (sums[i], sums[i+1])
        return 0

if __name__ == '__main__':
    main()