Schlagwort-Archive: python

Python: Snippet Multiprocessing mit Ergebnis

Beispiel für Parallelisierung von Jobs mit Ergebnis welche als Liste zurückgegeben werden.

#!/usr/bin/env python
# -*- encoding: utf-8; py-indent-offset: 4 -*-

import os
from multiprocessing import Pool


def worker(job):
    x, y = job
    result = x ** y
    return os.getpid(), result
  
if __name__ == '__main__':
    jobs = [(1, 2), (3, 4), (5, 6), (11, 12), (13, 14), (15, 16), (21, 22), (23, 24), (25, 26)]
    
    result_buffer = []
  
    pool = Pool(processes=5)
    
    for job in jobs:
        result_buffer.append(pool.apply_async(worker, args=(job,)))
    
    pool.close()
    pool.join()
  
    results = [r.get() for r in result_buffer]

    print results
  
    for pid, result in results:
        print "working pid was: %s" % pid
        print "result is: %s" % result
        print "---"

Beispiel Ergebnis:

$python mp_with_result.py

[(7992, 1), (7992, 81), (7992, 15625), (7992, 3138428376721L), (7992, 3937376385699289L), (7992, 6568408355712890625L), (7992, 122694327386105632949003612841L), (7992, 480250763996501976790165756943041L), (7992, 2220446049250313080847263336181640625L)]
working pid was: 7992
result is: 1
---
working pid was: 7992
result is: 81
---
working pid was: 7992
result is: 15625
---
working pid was: 7992
result is: 3138428376721
---
working pid was: 7992
result is: 3937376385699289
---
working pid was: 7992
result is: 6568408355712890625
---
working pid was: 7992
result is: 122694327386105632949003612841
---
working pid was: 7992
result is: 480250763996501976790165756943041
---
working pid was: 7992
result is: 2220446049250313080847263336181640625
---

 

Python: Experiment/Snippet – Komprimieren und löschen von Logfiles nach X Tagen

Ein Ansatz für Logverzeichnisse im Format /log/<yyyy>/<mm>/<dd>/<div. logsfiles>

#!/usr/bin/env python

import gzip
import shutil
import os
import datetime
import time

#############################################
# Config
#############################################

# Path of Logfiles
# Structure is /opt/log/<YYYY>/<MM>/<DD>/
gpath='/opt/log/'

# hold logs for x days
hold_time=180



#############################################

def get_immediate_subdirectories(a_dir):
    return [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))]

def delete_files(f):
    # delete file if older than hold time
    nowx = time.time()

    for file in os.listdir(f):    
        if os.stat(f+file).st_mtime < nowx - hold_time * 86400:
            f_path = f+file
            print "delete %s " % f_path
            os.remove(f_path)
            
    try:
        os.rmdir(f)
    except:
        pass
            
    
    
def compress_files(lpath):
    # Compress files
    print "Working on: " + lpath
    obj = os.listdir(lpath)
    for f in obj:
        if os.path.isfile(lpath+f) and ".gz" not in f:
            with open(lpath+f,'rb') as f_in:
                with gzip.open(lpath+f+".gz",'wb') as f_out:
                    shutil.copyfileobj(f_in, f_out)
                    os.remove(lpath+f)
            
#compress everything which ist older than now

now = datetime.datetime.now()
years = get_immediate_subdirectories(gpath)

for year in years:

    # delete empty directories
    if not os.listdir(gpath+year):
        os.rmdir(gpath+year)
    else:

        months = get_immediate_subdirectories(gpath+year)
    
        for month in months:

            # delete empty directories
            if not os.listdir(gpath+year+"/"+month):
                os.rmdir(gpath+year+"/"+month)

            else:
                days = get_immediate_subdirectories(gpath+year+"/"+month)
                        
                # Remove current day from compressing & cleaning
                if month == str(now.month) and year == str(now.year):

                    if len(str(now.day)):
                        now_day = "0%s" % now.day
                    else:
                        now_day = str(now.day)

                    days.remove(now_day)
        
                for day in days:
                    # delete empty directories
                    if not os.listdir(gpath+year+"/"+month+"/"+day+"/"):
                        os.rmdir(gpath+year+"/"+month+"/"+day+"/")
                    else:
                        # compress all files in folder
                        compress_files(gpath+year+"/"+month+"/"+day+"/")
                        
                        # delete old files
                        delete_files(gpath+year+"/"+month+"/"+day+"/")

 

Python: Snippet – Kaputten UTF-8 String reparieren

Ich habe aus der Datenbank einen String zurückbekommen der UTF-8 war aber falsch kodiert zurückgegeben wurde. So wurde aus Geschäftsstelle -> Gesch├ñftsstelle

Folgendes Snippet kann einen kaputten UTF-8 String neu auf UTF-8 kodieren:

name_kaputt = 'Gesch\xc3\xa4ftsstelle'

name = ''.join(chr(ord(c)) for c in name_kaputt).decode("utf-8")

print name_kaputt
print name

Ergebnis:

Python 2.7.13 (v2.7.13:a06454b1afa1, Dec 17 2016, 20:42:59) [MSC v.1500 32 bit (Intel)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>> name_kaputt = 'Gesch\xc3\xa4ftsstelle'
>>> name = ''.join(chr(ord(c)) for c in name_kaputt).decode("utf-8")
>>> print name_kaputt
Geschäftsstelle
>>> print name
Geschäftsstelle