Adding the library that translates atmospheric filenames to NDN names
* Provide API that takes schema files and filenames returns NDN names
* Checks name-fields with metadata and raises possible errors
Change-Id:Ie19faa73e194a726276abc320822694652f6d04c
Refs: #2714
diff --git a/lib/ndn_cmmap_translators/README.md b/lib/ndn_cmmap_translators/README.md
new file mode 100644
index 0000000..2b03783
--- /dev/null
+++ b/lib/ndn_cmmap_translators/README.md
@@ -0,0 +1,7 @@
+This is a python3 library that provides an interface for translating netCDF
+filenames to NDN names. The library reads the filenames, metadata and user
+input for the translation. User defined schemas are defined in a conf file.
+For a sample conf file, see etc/cmip5/cmip5.conf.
+
+== Installation ==
+export PYTHONPATH=$PYTHONPATH:<path_to_lib>
diff --git a/lib/ndn_cmmap_translators/__init__.py b/lib/ndn_cmmap_translators/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/ndn_cmmap_translators/__init__.py
diff --git a/lib/ndn_cmmap_translators/atmos2ndn_parser/__init__.py b/lib/ndn_cmmap_translators/atmos2ndn_parser/__init__.py
new file mode 100644
index 0000000..a3603f8
--- /dev/null
+++ b/lib/ndn_cmmap_translators/atmos2ndn_parser/__init__.py
@@ -0,0 +1,4 @@
+'''define the modules here so that they can be imported'''
+__all__ = ["user_conf_parser", "parser", "cmd_arg_parser"]
+
+
diff --git a/lib/ndn_cmmap_translators/atmos2ndn_parser/cmd_arg_parser.py b/lib/ndn_cmmap_translators/atmos2ndn_parser/cmd_arg_parser.py
new file mode 100644
index 0000000..cb1f2b7
--- /dev/null
+++ b/lib/ndn_cmmap_translators/atmos2ndn_parser/cmd_arg_parser.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+# -*- Mode:python; c-file-style:"gnu"; indent-tabs-mode:nil -*- */
+#
+# Copyright (c) 2015, Colorado State University.
+#
+# This file is part of ndn-atmos.
+#
+# ndn-atmos is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later version.
+#
+# ndn-atmos is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received copies of the GNU General Public License and GNU Lesser
+# General Public License along with ndn-atmos, e.g., in COPYING.md file. If not, see
+# <http://www.gnu.org/licenses/>.
+#
+# See AUTHORS.md for complete list of ndn-atmos authors and contributors.
+
+'''this module parsed the command line arguments and return an object with
+the data filename and config filename'''
+import argparse
+import sys, traceback
+import os
+
+class InputParser(object):
+ '''parse the command line arguments
+ * -c or --conf for naming schema files
+ * -f is for filename to translate
+ * -d if for directory containing the files to translate
+ '''
+
+ def __init__(self):
+ self.confFilepath = None
+ self.dataFilepath = None
+
+ def parse(self):
+ '''Parse the command line arguments and get file/directory name
+ to translate. Also, get the schema config filename'''
+
+ parser = argparse.ArgumentParser(description='Translates netcdf names\
+ to NDN names')
+ parser.add_argument("-c", "--conf", required=True, help='full path to\
+ name configuration file')
+
+ #the translator either takes a filename or a directory with files
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument("-f", "--filename", help='takes a netcdf filename')
+ group.add_argument("-d", "--datadir", help='If specified, converts all\
+ files in the specified directory')
+ args = parser.parse_args()
+
+ #assign values to filename and confFile
+ try:
+ self.confFilepath = args.conf
+ self.dataFilepath = args.filename or args.datadir
+
+ #check if config file exists
+ confFile = open(self.confFilepath)
+ confFile.close()
+
+ #check if data directory or datafile exists
+ if args.filename is not None:
+ dataF = open(args.filename)
+ dataF.close()
+
+ #if datadir does not exists, raise exception
+ elif args.datadir is not None and os.path.isdir(args.datadir) is False:
+ raise Exception("No such directory '%s'" %(args.datadir))
+
+ except (IOError, OSError):
+ traceback.print_exc(file=sys.stdout)
+ sys.exit(1)
+ return 0
diff --git a/lib/ndn_cmmap_translators/atmos2ndn_parser/conf_file_parser.py b/lib/ndn_cmmap_translators/atmos2ndn_parser/conf_file_parser.py
new file mode 100644
index 0000000..f2c2a02
--- /dev/null
+++ b/lib/ndn_cmmap_translators/atmos2ndn_parser/conf_file_parser.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+# -*- Mode:python; c-file-style:"gnu"; indent-tabs-mode:nil -*- */
+#
+# Copyright (c) 2015, Colorado State University.
+#
+# This file is part of ndn-atmos.
+#
+# ndn-atmos is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later version.
+#
+# ndn-atmos is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received copies of the GNU General Public License and GNU Lesser
+# General Public License along with ndn-atmos, e.g., in COPYING.md file. If not, see
+# <http://www.gnu.org/licenses/>.
+#
+# See AUTHORS.md for complete list of ndn-atmos authors and contributors.
+
+'''This is the config file parser module.
+Input = object with command line parameters.
+Output = list of components for different config sections'''
+import configparser
+import sys, traceback
+
+class ParseConf(object):
+ '''parses the name schema file and returns name mappings for translated
+ output'''
+ def __init__(self, confName=None):
+ self.confFile = confName
+ if self.confFile is None:
+ raise Exception("No configuration file given to parse config")
+ self.filenameMap = []
+ self.ndnNameMap = []
+ self.seperatorsMap = []
+ self.userDefinedConfDir = {}
+ self.translator = []
+
+ if __debug__:
+ print(self.confFile)
+
+ self.parser = configparser.SafeConfigParser()
+ self.parser.optionxform=str
+ self.parser.read(self.confFile)
+ self.fullConf = {}
+
+ #parser now contain a dictionary with the sections in conf
+ #iterate over them and store the name components in fullConf
+ try:
+ for sectionName in self.parser.sections():
+ self.conf = {}
+ for name, value in self.parser.items(sectionName):
+ self.conf[name] = value
+ self.fullConf[sectionName] = self.conf
+ if __debug__:
+ print(self.fullConf)
+ except (KeyError, TypeError):
+ raise error.with_traceback(sys.exc_info()[2])
+
+ def getMappings(self):
+ '''parses the schema file and provides name mappings'''
+ try:
+ self.filenameMap = self.fullConf['Name']['filenameMapping'].replace(" ", "").split(',')
+
+ self.ndnNameMap = self.fullConf['Name']['ndnMapping'].replace(" ", "").split(',')
+ # user defined components look like this
+ #activity:cmip5, subactivity:atmos, organization:csu, ensemble:r3i1p1
+ userDefinedConf = self.fullConf['Name']['userDefinedComps'].replace(" ", "").split(',')
+ for item in userDefinedConf:
+ key, value = item.split(":")
+ self.userDefinedConfDir[key] = [value]
+
+ self.seperatorsMap = self.fullConf['Name']['seperators'].replace(" ", "").split(',')
+
+ #reads which translator to use
+ self.translator = self.fullConf['Translator']['translator'].replace(" ", "")
+ except (KeyError, TypeError):
+ raise error.with_traceback(sys.exc_info()[2])
diff --git a/lib/ndn_cmmap_translators/atmos2ndn_translators/__init__.py b/lib/ndn_cmmap_translators/atmos2ndn_translators/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/ndn_cmmap_translators/atmos2ndn_translators/__init__.py
diff --git a/lib/ndn_cmmap_translators/atmos2ndn_translators/cmip5_translator.py b/lib/ndn_cmmap_translators/atmos2ndn_translators/cmip5_translator.py
new file mode 100644
index 0000000..2cc3ca6
--- /dev/null
+++ b/lib/ndn_cmmap_translators/atmos2ndn_translators/cmip5_translator.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+# -*- Mode:python; c-file-style:"gnu"; indent-tabs-mode:nil -*- */
+#
+# Copyright (c) 2015, Colorado State University.
+#
+# This file is part of ndn-atmos.
+#
+# ndn-atmos is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later version.
+#
+# ndn-atmos is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received copies of the GNU General Public License and GNU Lesser
+# General Public License along with ndn-atmos, e.g., in COPYING.md file. If not, see
+# <http://www.gnu.org/licenses/>.
+#
+# See AUTHORS.md for complete list of ndn-atmos authors and contributors.
+
+'''this is the cmip5 translator module'''
+
+import re
+import sys
+import netCDF4
+import os
+import traceback
+
+class Cmip5NameTranslator(object):
+ def __init__(self):
+ self.finalName = '/'
+
+ def checkGranularity(self, ncFile, fullFilePath):
+ '''check the granularity/time frequency of values from metatdata
+ and crosscheck by calculating it from data'''
+
+ #get first two values of time variable
+ time = ncFile.variables['time'][0:2]
+
+ #get time unit
+ unit = ncFile.variables['time'].units
+
+ #get granularity in metadata
+ metadataGranularity = getattr(ncFile, 'frequency')
+
+ #if day appears in unit, the index is based on day
+ if 'day' in unit:
+ timeFromData = str(int((time[1] - time[0])*24)) + 'hr'
+
+ #does this ever happen? Check
+ else:
+ print("Warning: Data not in day granularity")
+ return False
+
+ #if granularity from metadata and data does not match, error, else keep record
+ if timeFromData != metadataGranularity:
+ print("Error in file %s. Name component 'time' is '%s' in data and '%s' in metadata "
+ %(fullFilePath, timeFromData, metadataGranularity))
+ return False
+
+ else:
+ if __debug__:
+ print("file, time from data , time from metadata," ,ncFile, timeFromData,\
+ metadataGranularity)
+ return timeFromData
+
+ def translate(self, fullPath, parsedConfig):
+ '''translates netcdf filename to ndn name. For example,
+ psl_6hrPlev_MIROC5_historical_r3i1p1_1968010100-1968123118.nc becomes
+ /CMIP5/output/MIROC/MIROC5/historical/6hr/atmos/psl/r1i1p1/1968010100-1968123118/'''
+
+ fileName = fullPath.split('/')[-1]
+ fullFilePath = fullPath
+ filenameMap = parsedConfig.filenameMap
+ ndnNameMap = parsedConfig.ndnNameMap
+ seperatorsMap = parsedConfig.seperatorsMap
+ confName = parsedConfig.confFile
+ fileCompDict = {}
+ metadataCompDict = {}
+
+
+ #split the input filename, we will use this for ndn name create a regexp
+ combinedSeperators = '|'.join(map(re.escape, seperatorsMap))
+ splitFileName = re.split(combinedSeperators, fileName)
+ if __debug__:
+ print("Split file name", splitFileName)
+
+ #check if input file maps to conf mapping
+ if len(splitFileName) != len(filenameMap):
+ print("Error: Input file %s does not match schema described in configuration file in" \
+ "%s, skipping" %(fileName, confName))
+
+
+ #create a directory mapping DRS components to filename components
+ for item in range(len(filenameMap)):
+ fileCompDict[filenameMap[item].strip()] = splitFileName[item]
+ if __debug__:
+ print(fileCompDict)
+
+ #open the file and read the metadata
+ try:
+ with netCDF4.Dataset(fullFilePath, 'r') as ncFile:
+ #for each item in ndnMapping list, try getting it from metadata
+ for nameComp in ndnNameMap:
+ try:
+ metadataCompDict[nameComp.strip()] = getattr(ncFile, nameComp.strip())
+ except:
+ if __debug__:
+ print("'%s' not found in metadata" %(nameComp.strip()))
+ pass
+ if __debug__:
+ print(metadataCompDict)
+ print(ndnNameMap)
+ for item in ndnNameMap:
+ if (item == 'frequency'):
+ #get it from data, if frequency differs in metadata and data, throw error
+ freq = self.checkGranularity(ncFile, fullFilePath)
+ if freq is False:
+ self.finalName = ''
+ return False
+ else:
+ if item in metadataCompDict:
+ if item in metadataCompDict and item in fileCompDict:
+ if metadataCompDict[item] != fileCompDict[item]:
+ print("Error in file %s" %(fullFilePath))
+ print("Name component '%s' is '%s' in name and '%s' in metadata " \
+ %(item, fileCompDict[item], metadataCompDict[item]))
+ self.finalName = ''
+ return False
+ #create the name
+ if item in fileCompDict:
+ self.finalName += fileCompDict[item] + '/'
+ else:
+ try:
+ self.finalName += metadataCompDict[item] + '/'
+ except KeyError:
+ print("Error: %s does not have '%s' component needed for translation"
+ %(fileName, item))
+ traceback.print_exc(file=sys.stdout)
+ return False
+ except (RuntimeError, IOError):
+ print("%s is not a valid netCDF file" %(fullFilePath))
+ traceback.print_exc(file=sys.stdout)
+ return False
+ #if final name has any spaces, remove them
+ self.finalName = self.finalName.replace(" ", "")
+ return True
diff --git a/lib/ndn_cmmap_translators/atmos2ndn_translators/translate.py b/lib/ndn_cmmap_translators/atmos2ndn_translators/translate.py
new file mode 100644
index 0000000..fdd7628
--- /dev/null
+++ b/lib/ndn_cmmap_translators/atmos2ndn_translators/translate.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+# -*- Mode:python; c-file-style:"gnu"; indent-tabs-mode:nil -*- */
+#
+# Copyright (c) 2015, Colorado State University.
+#
+# This file is part of ndn-atmos.
+#
+# ndn-atmos is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later version.
+#
+# ndn-atmos is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received copies of the GNU General Public License and GNU Lesser
+# General Public License along with ndn-atmos, e.g., in COPYING.md file. If not, see
+# <http://www.gnu.org/licenses/>.
+#
+# See AUTHORS.md for complete list of ndn-atmos authors and contributors.
+
+'''Translates a netcdf filename to a NDN name'''
+
+import sys, traceback
+import configparser
+import re
+import netCDF4
+import glob
+import os
+from atmos2ndn_parser import cmd_arg_parser
+from atmos2ndn_parser import conf_file_parser
+from . import cmip5_translator
+
+def translate(parsedConfig, dataFilepath):
+ '''translate module is called by wrapper functions
+ with file/directory name and returns NDN name'''
+
+ translatedFileNames = []
+ filenameMap = parsedConfig.filenameMap
+ ndnNameMap = parsedConfig.ndnNameMap
+ seperatorsMap = parsedConfig.seperatorsMap
+ userDefinedMapping = parsedConfig.userDefinedConfDir
+ fullPath = dataFilepath
+ translator= parsedConfig.translator
+
+ if __debug__:
+ print("In translate")
+ print("Translator=", translator)
+
+ translatorFunction = None
+ if translator == 'cmip5_translator':
+ translatorFunction = cmip5_translator.Cmip5NameTranslator
+ else:
+ raise RuntimeError("Error: Invalid translator specified by user. \
+ Choice is 'cmip5_translator'")
+
+ if os.path.isdir(fullPath) is True:
+ #we have been given a directory, walk it
+ for root, subdirs, files in os.walk(fullPath):
+ if __debug__:
+ print('='*60)
+ print('Working on = ' + root)
+ print('='*60)
+
+ for fileName in files:
+ if __debug__:
+ print("fileName, filenameMap, ndnNameMap, \
+ seperatorsMap, userDefinedMapping", fileName, \
+ filenameMap, ndnNameMap, seperatorsMap, \
+ userDefinedMapping)
+ try:
+ #this is where translation happens
+ if os.path.isfile(os.path.join(root, fileName)):
+ if __debug__:
+ print("-"*80)
+ print("Original File Name: %s \n" %(fileName))
+ translateObj = translatorFunction()
+ res = translateObj.translate(os.path.join(root, fileName), parsedConfig)
+ if res:
+ if __debug__:
+ print("NDN Name: %s\n" %(translateObj.finalName))
+ print("-"*80)
+ translatedFileNames.append(translatedObj.finalName)
+
+ except Exception as err:
+ traceback.print_exc(file=sys.stdout)
+ #don't stop for a garbled file
+ pass
+ return translatedFileNames
+ #else work only on the given file
+ else:
+ try:
+ if __debug__:
+ print("fileName, filenameMap, ndnNameMap, seperatorsMap, userDefinedMapping",\
+ fullPath, filenameMap, ndnNameMap, seperatorsMap, userDefinedMapping)
+ root = os.path.dirname(fullPath)
+ fileName = os.path.split(fullPath)[-1]
+ if __debug__:
+ print("-"*80)
+ print("Original File Name: %s \n" %(fileName))
+ translateObj = translatorFunction()
+ res = translateObj.translate(fullPath, parsedConfig)
+ if res:
+ if __debug__:
+ print("NDN Name: %s\n" %(translateObj.finalName))
+ print("-"*80)
+ return [translateObj.finalName]
+
+ except Exception as err:
+ traceback.print_exc(file=sys.stdout)
+ return None
diff --git a/lib/ndn_cmmap_translators/atmos_translator.py b/lib/ndn_cmmap_translators/atmos_translator.py
new file mode 100644
index 0000000..2c6a380
--- /dev/null
+++ b/lib/ndn_cmmap_translators/atmos_translator.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+# -*- Mode:python; c-file-style:"gnu"; indent-tabs-mode:nil -*- */
+#
+# Copyright (c) 2015, Colorado State University.
+#
+# This file is part of ndn-atmos.
+#
+# ndn-atmos is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later version.
+#
+# ndn-atmos is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received copies of the GNU General Public License and GNU Lesser
+# General Public License along with ndn-atmos, e.g., in COPYING.md file. If not, see
+# <http://www.gnu.org/licenses/>.
+#
+# See AUTHORS.md for complete list of ndn-atmos authors and contributors.
+
+'''Translates a netcdf filename to a NDN name'''
+
+from atmos2ndn_parser import conf_file_parser, cmd_arg_parser
+from atmos2ndn_translators import translate
+
+def argsForTranslation(dataFilepath, configPath):
+ '''this module does the actual translation calls'''
+ if __debug__:
+ print("config file '%s', data path '%s'" %(configPath, dataFilepath))
+
+ #pass the config file to parser module, which will return and object
+ #with all the mappings. The mappings tell us which name component
+ #comes from where and what should be the order of the components
+ parsedConfig = conf_file_parser.ParseConf(configPath)
+ res = parsedConfig.getMappings()
+
+ #do the translation
+ ndnName = translate.translate(parsedConfig, dataFilepath)
+ if __debug__:
+ print("NDN name in main: %s" %(ndnName))
+
+def main():
+ '''parse command line arguments, gives back configFilename and dataFilename
+ we then pass these to the subsequent modules.'''
+
+ userArgs = cmd_arg_parser.InputParser()
+ userArgs.parse()
+ configFile = userArgs.confFilepath
+ print("config file '%s', data path '%s'" %(configFile, userArgs.dataFilepath))
+ if __debug__:
+ print("config file '%s', data path '%s'" %(configFile, userArgs.dataFilepath))
+
+ #call the translator module
+ argsForTranslation(userArgs.dataFilepath, configFile)
+
+if __name__ == '__main__':
+ main()
diff --git a/lib/ndn_cmmap_translators/etc/cmip5/cmip5.conf b/lib/ndn_cmmap_translators/etc/cmip5/cmip5.conf
new file mode 100644
index 0000000..6b16ef9
--- /dev/null
+++ b/lib/ndn_cmmap_translators/etc/cmip5/cmip5.conf
@@ -0,0 +1,21 @@
+[Name]
+
+#ndn_mapping = activity, subactivity, organization, experiment, granularity, model_id, variable_name, ensemble
+
+ndnMapping = project_id, product, institute_id, model_id, experiment_id, frequency, modeling_realm, variable_name, parent_experiment_rip, start_time
+
+#fullName = psl_6hrPlev_bcc-csm1-1_historical_r3i1p1_198001010000-201212311800.nc
+
+filenameMapping = variable_name, mip_table, model_id, experiment_id, parent_experiment_rip, start_time, filetype
+
+seperators = _,.
+
+compsFromData = frequency
+
+#compsFromMetadata = project_id, product, institute_id, model_id, experiment_id, frequency, modeling_realm
+
+userDefinedComps = activity:cmip5, subactivity:atmos, organization:csu, parent_experiment_rip:r3i1p1
+
+[Translator]
+translator = cmip5_translator
+
diff --git a/lib/ndn_cmmap_translators/setup.py b/lib/ndn_cmmap_translators/setup.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/ndn_cmmap_translators/setup.py
diff --git a/lib/ndn_cmmap_translators/tests/__init__.py b/lib/ndn_cmmap_translators/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/ndn_cmmap_translators/tests/__init__.py
diff --git a/lib/ndn_cmmap_translators/tests/ndn-atmos-translator_tests.py b/lib/ndn_cmmap_translators/tests/ndn-atmos-translator_tests.py
new file mode 100644
index 0000000..6771494
--- /dev/null
+++ b/lib/ndn_cmmap_translators/tests/ndn-atmos-translator_tests.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+# -*- Mode:python; c-file-style:"gnu"; indent-tabs-mode:nil -*- */
+#
+# Copyright (c) 2015, Colorado State University.
+#
+# This file is part of ndn-atmos.
+#
+# ndn-atmos is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later version.
+#
+# ndn-atmos is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received copies of the GNU General Public License and GNU Lesser
+# General Public License along with ndn-atmos, e.g., in COPYING.md file. If not, see
+# <http://www.gnu.org/licenses/>.
+#
+# See AUTHORS.md for complete list of ndn-atmos authors and contributors.
+
+import unittest
+import inspect
+import argparse
+import ConfigParser
+import sys
+
+sys.path.append("../")
+sys.path.append("../../")
+
+import atmos2ndn_translators
+from atmos2ndn_translators.parser import parser, user_conf_parser
+from netcdf2ndn.translators import core_translator, cam_translator
+
+
+class TestCommandParser(unittest.TestCase):
+
+ maxDiff = None
+ def setUp(self):
+ self.parse_file = None
+ self.parsed_config = None
+ self.test_conf_file = "netcdf2ndn/etc/cesm-0.conf"
+ self.user_conf_file = "netcdf2ndn/etc/user_spec_comps.conf"
+ self.filename_mapping = ''
+ self.parse_file = parser.parse_conf(self.test_conf_file)
+ self.parse_user_file = user_conf_parser.parse_user_conf(self.user_conf_file)
+
+# def CmdArgParse(self):
+# don't know how to test
+
+ def TestFilenameMapping(self):
+ '''test original filename mapping is fine'''
+ parsed = self.parse_file.filename_mapping()
+ expected_filename_mapping=['experiment', 'experiment', 'model', 'granularity', 'year', 'month', 'day',
+ 'time', 'filetype']
+ print parsed, expected_filename_mapping
+ self.assertEqual(expected_filename_mapping, parsed)
+
+ def TestNDNNameMapping(self):
+ '''test ndn name component mapping is fine'''
+ parsed = self.parse_file.ndn_name_mapping()
+ expected_ndn_name_mapping=['activity', 'subactivity', 'organization', 'ensemble', 'experiment',
+ 'model', 'granularity', 'start_time']
+ self.assertEqual(expected_ndn_name_mapping, parsed)
+
+ def TestSeperatorsMapping(self):
+ '''test seperators mapping is parsed fine'''
+ parsed = self.parse_file.seperators_mapping()
+ expected_seperators_mapping=['_', '.', '-']
+ self.assertEqual(expected_seperators_mapping, parsed)
+
+ def TestCompFromDataMapping(self):
+ '''test component from data mapping is parsed fine'''
+ parsed = self.parse_file.comp_from_data()
+ expected_comp_mapping=['activity', 'subactivity', 'organization', 'ensemble', 'granularity', 'start_time']
+ self.assertEqual(expected_comp_mapping, parsed)
+
+ def TestParseUserConf(self):
+ '''test correct parsing of user defined mappings'''
+ parsed = self.parse_user_file.user_defined_mapping()
+ print parsed
+ expected_mapping = {'organization': ['csu'], 'granularity': [''], 'ensemble': ['test'], 'subactivity': ['atmos'], 'activity': ['gcrm']}
+ self.assertEqual(expected_mapping, parsed)
+
+ def TestDoTranslate(self):
+ '''test translation function'''
+ expected_translated_name = {'orig_name': 'spcesm-ctrl.cam2.h0.1891-01.nc', 'final_ndn_name_str':
+ '/gcrm/atmos/csu/test/spcesm-ctrl/cam2/1M/1891-01/'}
+ translated_name = core_translator.do_translate('/home/susmit/atmos_filesystem/CESM_sample_output/spcesm-ctrl.cam2.h0.1891-01.nc',
+ ['experiment', 'stream', 'granularity', 'start_time', 'filetype'], ['activity', 'subactivity', 'organization','ensemble', 'experiment', 'stream', 'granularity', 'start_time'], ['_', '.'], {'organization': ['csu'], 'granularity': [''],'ensemble': ['test'], 'subactivity': ['atmos'], 'activity': ['gcrm']})
+ self.assertEqual(expected_translated_name, translated_name)
+
+
+if __name__ == '__main__':
+ unittest.main()
+