add new compress script lz4.py

compress_logger/compress.py

@@ -26,21 +26,21 @@ def zcat_compress(gzfile, files):
 def zcat_expand(gzfile):
     with gzip.open(gzfile) as f:
         line = f.readline()
-        assert(line.startswith('ZCAT'))
+        assert(line.startswith(b'ZCAT'))
         line = f.readline()
         n = 0
         m = 0
-        while line != '':
-            size, filename = line[:-1].split(' ',1)
+        while line != b'':
+            size, filename = line[:-1].split(b' ',1)
             content = f.read(int(size))
             try:
-                with open(filename) as g:
+                with open(filename, 'rb') as g:
                     c = g.read()
                 if content != c:
                     print('%s does not match' % filename)
                 m += 1
             except IOError:
-                with open(filename, 'w') as g:
+                with open(filename, 'wb') as g:
                     g.write(content)
                 n += 1
             line = f.readline()
@@ -51,7 +51,7 @@ def instdir(instrument=''):
     if instrument != '':
         instrument += '/'
     return '%s/sea/%s' % (os.path.expanduser("~"), instrument)
-        
+
 def compress(logdir, gzdir, variable):
     os.chdir(logdir + variable)
     if not os.path.isdir(gzdir):
@@ -92,7 +92,10 @@ def treat(action, instrument='', years='past', vars=None):
     for year in yrs:
         logdir = logbase + year + '/'
         gzdir = gzbase + year + '/'
-        os.chdir(logdir)
+        if action == COMPRESS:
+            os.chdir(logdir)
+        else:
+            os.chdir(gzdir)
         if not vars:
             varlist = list(glob('*'))
         else:
@@ -105,7 +108,9 @@ def treat(action, instrument='', years='past', vars=None):
                         compress(logdir, gzdir, variable)
                         time.sleep(min(1.0, time.time()-t0)) # be nice
             elif action == EXPAND:
-                os.makedirs(logdir+variable)
+                if variable.endswith('.gz'):
+                    variable = variable[:-3]
+                os.makedirs(logdir+variable, exist_ok=True)
                 print(gzdir, variable, '.gz')
                 if os.path.isfile(gzdir + variable + '.gz'):
                     expand(logdir, gzdir, variable)