Record missing values for each variable according to EBAS value convention

This commit is contained in:
2025-05-21 13:53:18 +02:00
parent e4b2a4cd5a
commit bd74f8310c

View File

@ -75,6 +75,22 @@ def extract_var_descriptions(part2):
return descriptions
def generate_missing_value_code(max_val, num_decimals):
"""
Generates a missing value code consisting of all 9s.
- `max_val`: Largest expected valid value in the column.
- `num_decimals`: Number of decimal places to preserve.
"""
# Determine order of magnitude (1-2 orders larger than max value)
order = int(np.floor(np.log10(max_val))) + 2 if max_val > 0 else 2
# Construct the missing value code as all 9s
if num_decimals > 0:
missing_code = float(f"{'9' * (order + num_decimals)}.{ '9' * num_decimals }")
else:
missing_code = int('9' * order)
return missing_code
@ -157,6 +173,11 @@ def read_nasa_ames_as_dict(filename, instruments_dir: str = None, work_with_copy
if 'end_time' in df.columns:
df['end_time'] = df['end_time'].apply(lambda x: start_date + timedelta(days=x))
variable_missing_values = []
for col in df.columns:
missing_value = generate_missing_value_code(df[col].max(skipna=True), 4)
variable_missing_values.append(missing_value)
# Create header metadata dictionary
header_metadata_dict = {
'header_length': header_length,
@ -164,6 +185,7 @@ def read_nasa_ames_as_dict(filename, instruments_dir: str = None, work_with_copy
#'num_dep_vars': num_dep_vars,
'variable_names': vars_list,
'variable_descriptions' : var_descriptions,
'variable_missing_values': variable_missing_values,
'raw_header_part1': part1,
'raw_header_part2': part2,
'raw_header_part3': part3