Python: Experiment/Snippet – Komprimieren und löschen von Logfiles nach X Tagen

Ein Ansatz für Logverzeichnisse im Format /log/<yyyy>/<mm>/<dd>/<div. logsfiles>

#!/usr/bin/env python

import gzip
import shutil
import os
import datetime
import time

#############################################
# Config
#############################################

# Path of Logfiles
# Structure is /opt/log/<YYYY>/<MM>/<DD>/
gpath='/opt/log/'

# hold logs for x days
hold_time=180



#############################################

def get_immediate_subdirectories(a_dir):
    return [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))]

def delete_files(f):
    # delete file if older than hold time
    nowx = time.time()

    for file in os.listdir(f):    
        if os.stat(f+file).st_mtime < nowx - hold_time * 86400:
            f_path = f+file
            print "delete %s " % f_path
            os.remove(f_path)
            
    try:
        os.rmdir(f)
    except:
        pass
            
    
    
def compress_files(lpath):
    # Compress files
    print "Working on: " + lpath
    obj = os.listdir(lpath)
    for f in obj:
        if os.path.isfile(lpath+f) and ".gz" not in f:
            with open(lpath+f,'rb') as f_in:
                with gzip.open(lpath+f+".gz",'wb') as f_out:
                    shutil.copyfileobj(f_in, f_out)
                    os.remove(lpath+f)
            
#compress everything which ist older than now

now = datetime.datetime.now()
years = get_immediate_subdirectories(gpath)

for year in years:

    # delete empty directories
    if not os.listdir(gpath+year):
        os.rmdir(gpath+year)
    else:

        months = get_immediate_subdirectories(gpath+year)
    
        for month in months:

            # delete empty directories
            if not os.listdir(gpath+year+"/"+month):
                os.rmdir(gpath+year+"/"+month)

            else:
                days = get_immediate_subdirectories(gpath+year+"/"+month)
                        
                # Remove current day from compressing & cleaning
                if month == str(now.month) and year == str(now.year):

                    if len(str(now.day)):
                        now_day = "0%s" % now.day
                    else:
                        now_day = str(now.day)

                    days.remove(now_day)
        
                for day in days:
                    # delete empty directories
                    if not os.listdir(gpath+year+"/"+month+"/"+day+"/"):
                        os.rmdir(gpath+year+"/"+month+"/"+day+"/")
                    else:
                        # compress all files in folder
                        compress_files(gpath+year+"/"+month+"/"+day+"/")
                        
                        # delete old files
                        delete_files(gpath+year+"/"+month+"/"+day+"/")