diff --git a/compress_logger/compress.py b/compress_logger/compress.py index 50a35d8..4a880e4 100644 --- a/compress_logger/compress.py +++ b/compress_logger/compress.py @@ -26,21 +26,21 @@ def zcat_compress(gzfile, files): def zcat_expand(gzfile): with gzip.open(gzfile) as f: line = f.readline() - assert(line.startswith('ZCAT')) + assert(line.startswith(b'ZCAT')) line = f.readline() n = 0 m = 0 - while line != '': - size, filename = line[:-1].split(' ',1) + while line != b'': + size, filename = line[:-1].split(b' ',1) content = f.read(int(size)) try: - with open(filename) as g: + with open(filename, 'rb') as g: c = g.read() if content != c: print('%s does not match' % filename) m += 1 except IOError: - with open(filename, 'w') as g: + with open(filename, 'wb') as g: g.write(content) n += 1 line = f.readline() @@ -51,7 +51,7 @@ def instdir(instrument=''): if instrument != '': instrument += '/' return '%s/sea/%s' % (os.path.expanduser("~"), instrument) - + def compress(logdir, gzdir, variable): os.chdir(logdir + variable) if not os.path.isdir(gzdir): @@ -92,7 +92,10 @@ def treat(action, instrument='', years='past', vars=None): for year in yrs: logdir = logbase + year + '/' gzdir = gzbase + year + '/' - os.chdir(logdir) + if action == COMPRESS: + os.chdir(logdir) + else: + os.chdir(gzdir) if not vars: varlist = list(glob('*')) else: @@ -105,7 +108,9 @@ def treat(action, instrument='', years='past', vars=None): compress(logdir, gzdir, variable) time.sleep(min(1.0, time.time()-t0)) # be nice elif action == EXPAND: - os.makedirs(logdir+variable) + if variable.endswith('.gz'): + variable = variable[:-3] + os.makedirs(logdir+variable, exist_ok=True) print(gzdir, variable, '.gz') if os.path.isfile(gzdir + variable + '.gz'): expand(logdir, gzdir, variable) diff --git a/compress_logger/lz4.py b/compress_logger/lz4.py new file mode 100644 index 0000000..71bf363 --- /dev/null +++ b/compress_logger/lz4.py @@ -0,0 +1,104 @@ +from glob import glob +import re +import os +import sys +import shutil +import time + + +def instdir(instrument=''): + if instrument != '': + instrument += '/' + return '%s/sea/%s' % (os.path.expanduser("~"), instrument) + + +COMPRESS = 1 +EXPAND = 2 + + +def treat(instrument, yearlimit): + logbase = '%slogger/' % instdir(instrument) + os.chdir(logbase) + print('chdir', logbase) + os.makedirs('trash', exist_ok=True) + for ynum in range(2007, 2040): + year = str(ynum) + cyear = f'{year}.tar.lz4' + if year < yearlimit: + if os.path.exists(year): + if os.path.exists(cyear): + print(cyear, 'exists already') + else: + os.system(f'tar cf - {year} | lz4 - {cyear}') + print(cyear, 'created') + if os.path.exists(f'trash/{year}'): + shutil.rmtree(f'trash/{year}') + os.rename(year, f'trash/{year}') + else: + print('no', year) + else: + if os.path.exists(cyear): + if os.path.exists(year): + print(year, 'exists already') + else: + os.system(f'lz4 -d {cyear} | tar xf -') + print(year, 'created') + os.rename(cyear, f'trash/{cyear}') + + +if __name__ == '__main__': + action = COMPRESS + year = time.strftime("%Y") + instruments = [] + cleanup = True + for opt in sys.argv[1:]: + if opt == '-x': + action = EXPAND + cleanup = False + elif opt == '-c': + cleanup = False + else: + if opt.startswith('2'): + year = opt + else: + instruments.append(opt) + if len(instruments) == 0: + instruments = [''] + for instrument in instruments: + if not os.path.isdir(instdir(instrument)): + raise ValueError('%s is not an instrument on this machine' % instrument) + removedirs = ( + 'device.force_status_save', + 'nv.autoflow.*buf', + 'nv.autoflow.*min', + 'nv.autoflow.*max', + 'table.val_*', + '*.*.stddev', + 'device.ccu4version', + 'cc.f', + 'cc.nfb', + 'cc.hfb', + 'graph_*', + ) + print("instruments: %s" % ", ".join(instruments)) + print("compress years before %s" % year) + for instrument in instruments: + print('instrument:', instdir(instrument)) + if cleanup: + for pat in removedirs: + print('remove', pat) + gap = 0 + t0 = time.time() + t1 = t0 + for dir in glob(instdir(instrument)+'logger/*/'+pat): + shutil.rmtree(dir) + now = time.time() + if now > t0 + 5: + if gap > 0: + print("+%d" % gap) + print(dir) + else: + gap += 1 + time.sleep(min(1.0, now - t1)) # be nice + t1 = now + treat(instrument, year)