Changeset 4758 for palm/trunk/SCRIPTS/palm_cvd

Timestamp:

Oct 26, 2020 1:03:52 PM (4 years ago)

Author:

suehring

Message:

script to create input file for virtual measurements - palm_cvd: In order to do not omit observations that are on the same site but have different oordinates or feature-types, process all files rather than only one and omit the rest

File:

• palm/trunk/SCRIPTS/palm_cvd (modified) (6 diffs)

palm/trunk/SCRIPTS/palm_cvd

@@ -26 +26 @@
 # -----------------
 # $Id$
+# In order to do not omit observations that are on the same site but have different
+# coordinates or feature-types, process all files rather than only one and omit 
+# the rest.
+#
+# 4663 2020-09-02 14:54:09Z gronemeier
 # bugfix in non_measurable_vars; ignore station_h if featureType is trajectory
 #
@@ -198 +203 @@
 name_fill          = "_FillValue"
 name_site          = "site"
+name_acro          = "acronym"
+name_content       = "data_content"
 name_orig_x        = "origin_x"
 name_orig_y        = "origin_y"
@@ -268 +275 @@
    # virtual measurement has an overhead and consumes memory.
    sites = []
+   input_files = []
    for dirname in list_input_data:
        data_file = data_path + dirname
@@ -284 +292 @@
        input_file = data_file + "/" + latest_file
        ncfile_in = Dataset( input_file, "r", format="NETCDF4", encoding='ascii')
-
-       # Read global attributes and write them immediately into the output file
-       for att in ncfile_in.ncattrs():
-          if ( att == name_site ):
-             site = ncfile_in.getncattr(att)
-
-       if ( site not in sites ):
-          sites.append(site)
-
-   # Define a flag array that is used to identify whether site dimensions are already
-   # defined or not.
-   create_metadata_for_site = [None] * len(sites)
+       input_files.append(input_file)
 
    # Define a nested list of default variables that shall be measured. Based on this list,
    # the final number of measured variables is determined.
-   measured_variables_all_sites = [ ['u', 'v', 'w', 'theta', 'hus'] for var in range(0, len(sites))]
+   measured_variables_all_sites = [ ['u', 'v', 'w', 'theta', 'hus'] for var in range(0, len(input_files))]
 
    # Run loop over all subdirectories that contain observational data
-   for dirname in list_input_data:
-      data_file = data_path + dirname
-
-      # Directory may contain various file versions.
-      # Take the one with highest cycle number.
-      highest_cycle_nr = 0
-      for filename in os.listdir(data_file):
-         start_seq = len( filename ) - 6
-         end_seq   = len( filename ) - 3
-         if int( filename[start_seq:end_seq] ) > highest_cycle_nr:
-            highest_cycle_nr = int(filename[start_seq:end_seq])
-            latest_file      = filename
+   for counter, file in enumerate(input_files):
 
       # Open the NetCDF input file
-      input_file = data_file + "/" + latest_file
+      input_file = input_files[counter]
       ncfile_in = Dataset( input_file, "r", format="NETCDF4", encoding='ascii' )
 
-      # Read site attribue first
+      # Determine index for the treated site
+      num_vmeas = input_files.index( input_file ) + 1
+      print( counter, num_vmeas )
+
+      # Read global attributes and write them immediately into the output file
       for att in ncfile_in.ncattrs():
+         if ( att == name_featuretype ):
+            feature = ncfile_in.getncattr(att)
+         if ( att == name_datacontent ):
+            content = ncfile_in.getncattr(att)
          if ( att == name_site ):
             site = ncfile_in.getncattr(att)
 
-      # Determine index for the treated site
-      num_vmeas = sites.index( site ) + 1
-
-      # Check whether metadata for this site has been already created
-      if ( create_metadata_for_site[sites.index( site )] != "Done" ):
-
-         # Read global attributes and write them immediately into the output file
-         for att in ncfile_in.ncattrs():
-            if ( att == name_featuretype ):
-               feature = ncfile_in.getncattr(att)
-            if ( att == name_datacontent ):
-               content = ncfile_in.getncattr(att)
-            if ( att == name_site ):
-               site = ncfile_in.getncattr(att)
-
-            if ( att in atts_float ):
-               ncfile_out.setncattr( att + str(num_vmeas), np.double(ncfile_in.getncattr(att)) )
+         if ( att in atts_float ):
+            ncfile_out.setncattr( att + str(num_vmeas), np.double(ncfile_in.getncattr(att)) )
+         else:
+            ncfile_out.setncattr( att + str(num_vmeas), ncfile_in.getncattr(att) )
+
+      #timeSeries
+      if ( feature == name_ts ):
+         ntime = len( ncfile_in.dimensions[name_ntime]   )
+         nstat = len( ncfile_in.dimensions[name_station] )
+         ncfile_out.createDimension( name_ntime   + str(num_vmeas), ntime )
+         ncfile_out.createDimension( name_station + str(num_vmeas), nstat )
+
+      #trajectory
+      elif ( feature == name_traj ):
+         ntime = len( ncfile_in.dimensions[name_ntime]   )
+         ntraj = len( ncfile_in.dimensions[name_traj_dim] )
+         ncfile_out.createDimension( name_ntime    + str(num_vmeas), ntime )
+         ncfile_out.createDimension( name_traj_dim + str(num_vmeas), ntraj )
+
+      #timeseriesProfile
+      else:
+         ntime = len( ncfile_in.dimensions[name_ntime]   )
+         nstat = len( ncfile_in.dimensions[name_station] )
+         nz    = len( ncfile_in.dimensions[name_nz]      )
+         ncfile_out.createDimension( name_ntime   + str(num_vmeas), ntime )
+         ncfile_out.createDimension( name_station + str(num_vmeas), nstat )
+         ncfile_out.createDimension( name_nz      + str(num_vmeas), nz    )
+
+      for var in ncfile_in.variables.keys():
+         if ( var in dims_out ):
+            # Create a variable and write it to file after it is read. In order to
+            # avoid fill values in the dimensions, these are converted to zero
+            # before written to file. Depending on the featureType of the measurement,
+            # the array shape is different. For more informations, please see
+            # [UC]2 data standard.
+            # Timeseries
+            if ( feature == name_ts  ):
+               temp_ts = ncfile_out.createVariable( var + str(num_vmeas), float, \
+                                                    name_station + str(num_vmeas))
+               temp_ts[:] = np.nan_to_num( ncfile_in.variables[var][:] )
+
+            # Trajectories
+            elif ( feature == name_traj ):
+               # @note: If there are files where station_h is present although featureType is
+               #        trajectory, station_h must not be read.
+               if var != name_station_h:
+                  temp_traj = ncfile_out.createVariable( var + str(num_vmeas), float, \
+                                                         ( name_traj_dim + str(num_vmeas), \
+                                                           name_ntime + str(num_vmeas) ) )
+                  temp_traj[:,:] = np.nan_to_num( ncfile_in.variables[var][:,:] )
+
+            # TimeseriesProfiles
             else:
-               ncfile_out.setncattr( att + str(num_vmeas), ncfile_in.getncattr(att) )
-
-         #timeSeries
-         if ( feature == name_ts ):
-            ntime = len( ncfile_in.dimensions[name_ntime]   )
-            nstat = len( ncfile_in.dimensions[name_station] )
-            ncfile_out.createDimension( name_ntime   + str(num_vmeas), ntime )
-            ncfile_out.createDimension( name_station + str(num_vmeas), nstat )
-
-         #trajectory
-         elif ( feature == name_traj ):
-            ntime = len( ncfile_in.dimensions[name_ntime]   )
-            ntraj = len( ncfile_in.dimensions[name_traj_dim] )
-            ncfile_out.createDimension( name_ntime    + str(num_vmeas), ntime )
-            ncfile_out.createDimension( name_traj_dim + str(num_vmeas), ntraj )
-
-         #timeseriesProfile
-         else:
-            ntime = len( ncfile_in.dimensions[name_ntime]   )
-            nstat = len( ncfile_in.dimensions[name_station] )
-            nz    = len( ncfile_in.dimensions[name_nz]      )
-            ncfile_out.createDimension( name_ntime   + str(num_vmeas), ntime )
-            ncfile_out.createDimension( name_station + str(num_vmeas), nstat )
-            ncfile_out.createDimension( name_nz      + str(num_vmeas), nz    )
-
-         for var in ncfile_in.variables.keys():
-            if ( var in dims_out ):
-               # Create a variable and write it to file after it is read. In order to
-               # avoid fill values in the dimensions, these are converted to zero
-               # before written to file. Depending on the featureType of the measurement,
-               # the array shape is different. For more informations, please see
-               # [UC]2 data standard.
-               # Timeseries
-               if ( feature == name_ts  ):
-                  temp_ts = ncfile_out.createVariable( var + str(num_vmeas), float, \
+               if ( var == 'z' ):
+                  temp_pr = ncfile_out.createVariable( var + str(num_vmeas), float, \
+                                                      ( name_station + str(num_vmeas), \
+                                                        name_nz + str(num_vmeas) ) )
+                  temp_pr[:] = np.nan_to_num( ncfile_in.variables[var][:,0,:] )
+               else:
+                  temp_pr = ncfile_out.createVariable( var + str(num_vmeas), float, \
                                                        name_station + str(num_vmeas))
-                  temp_ts[:] = np.nan_to_num( ncfile_in.variables[var][:] )
-
-               # Trajectories
-               elif ( feature == name_traj ):
-                  # @note: If there are files where station_h is present although featureType is
-                  #        trajectory, station_h must not be read.
-                  if var != name_station_h:
-                     temp_traj = ncfile_out.createVariable( var + str(num_vmeas), float, \
-                                                            ( name_traj_dim + str(num_vmeas), \
-                                                              name_ntime + str(num_vmeas) ) )
-                     temp_traj[:,:] = np.nan_to_num( ncfile_in.variables[var][:,:] )
-
-               # TimeseriesProfiles
-               else:
-                  if ( var == 'z' ):
-                     temp_pr = ncfile_out.createVariable( var + str(num_vmeas), float, \
-                                                         ( name_station + str(num_vmeas), \
-                                                           name_nz + str(num_vmeas) ) )
-                     temp_pr[:] = np.nan_to_num( ncfile_in.variables[var][:,0,:] )
-                  else:
-                     temp_pr = ncfile_out.createVariable( var + str(num_vmeas), float, \
-                                                          name_station + str(num_vmeas))
-                     temp_pr[:] = np.nan_to_num( ncfile_in.variables[var][:] )
+                  temp_pr[:] = np.nan_to_num( ncfile_in.variables[var][:] )
 
       # Search for variables to be measured. In case the variable isn't already defined,
@@ -407 +386 @@
          if ( var not in non_measurable_vars  and  \
               var not in vars_default         and  \
-              var not in measured_variables_all_sites[sites.index( site )] ):
-
-            measured_variables_all_sites[sites.index( site )].append(var)
+              var not in measured_variables_all_sites[input_files.index( input_file )] ):
+
+            measured_variables_all_sites[input_files.index( input_file )].append(var)
 
       # Close the NetCDF input file
       ncfile_in.close()
 
-      # Set flag to indicate that for this specific site dimensions have been
-      # already created and attributes are already set.
-      if ( create_metadata_for_site[sites.index( site )] != "Done" ):
-         create_metadata_for_site[sites.index( site )] = "Done"
-
    # After variables are gathered and dimensions / attributes are already written to file,
    # the list of measured variables is written to file.
-   for site in sites:
-
-      num_vmeas = sites.index( site ) + 1
+   for site in input_files:
+
+      num_vmeas = input_files.index( site ) + 1
 
       ncfile_out.createDimension( "nvar"+ str(num_vmeas), \
-                                  len( measured_variables_all_sites[sites.index( site )] ) )
+                                  len( measured_variables_all_sites[input_files.index( site )] ) )
 
       measured = ncfile_out.createVariable( 'measured_variables' + str(num_vmeas), 'S1', \
                                             ("nvar" + str(num_vmeas), "string_len")) # must be NC_CHAR
 
-      for counter, meas in enumerate( measured_variables_all_sites[sites.index( site )] ):
+      for counter, meas in enumerate( measured_variables_all_sites[input_files.index( site )] ):
          measured[counter] = stringtochar( np.array( meas,"S%s"%(max_string_len) ) )
 
       # Check if any of the measured variables is a soil variable. Set flag accordingly.
       soil = False
-      for var in measured_variables_all_sites[sites.index( site )]:
+      for var in measured_variables_all_sites[input_files.index( site )]:
          if ( var in soil_vars ):
             soil = True
@@ -446 +420 @@
 
     # Store the number of observational sites
-   num_sites += len( sites )
+   num_sites += len( input_files )