|
|
# -*- coding: utf-8 -*-################################################################################ OpenERP, Open Source Management Solution# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).## This program is free software: you can redistribute it and/or modify# it under the terms of the GNU Affero General Public License as# published by the Free Software Foundation, either version 3 of the# License, or (at your option) any later version.## This program is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the# GNU Affero General Public License for more details.## You should have received a copy of the GNU Affero General Public License# along with this program. If not, see <http://www.gnu.org/licenses/>.###############################################################################import pprintimport mapperfrom openerp.tools.translate import _
import datetimeimport loggingimport StringIOimport tracebackpp = pprint.PrettyPrinter(indent=4)
class import_framework(): """
This class should be extends, get_data and get_mapping have to extends get_state_map and initialize can be extended for advanced purpose get_default_hook can also be extended @see dummy import for a minimal exemple """
"""
for import_object, this domain will avoid to find an already existing object """
DO_NOT_FIND_DOMAIN = [('id', '=', 0)]
#TODO don't use context to pass credential parameters def __init__(self, obj, cr, uid, instance_name, module_name, context=None): self.external_id_field = 'id' self.obj = obj self.cr = cr self.uid = uid self.instance_name = instance_name self.module_name = module_name self.context = context or {} self.table_list = [] self.logger = logging.getLogger(module_name) self.initialize()
"""
Abstract Method to be implemented in the real instance """
def initialize(self): """
init before import usually for the login """
pass
def init_run(self): """
call after intialize run in the thread, not in the main process TO use for long initialization operation """
pass
def get_data(self, table): """
@return: a list of dictionaries each dictionnaries contains the list of pair external_field_name : value """
return [{}]
def get_link(self, from_table, ids, to_table): """
@return: a dictionaries that contains the association between the id (from_table) and the list (to table) of id linked """
return {}
def get_external_id(self, data): """
@return the external id the default implementation return self.external_id_field (that has 'id') by default if the name of id field is different, you can overwrite this method or change the value of self.external_id_field """
return data[self.external_id_field]
def get_mapping(self): """
@return: { TABLE_NAME : { 'model' : 'openerp.model.name', #if true import the table if not just resolve dependencies, use for meta package, by default => True #Not required 'import' : True or False, #Not required 'dependencies' : [TABLE_1, TABLE_2], #Not required 'hook' : self.function_name, #get the val dict of the object, return the same val dict or False 'map' : { @see mapper 'openerp_field_name' : 'external_field_name', or val('external_field_name') 'openerp_field_id/id' : ref(TABLE_1, 'external_id_field'), #make the mapping between the external id and the xml on the right 'openerp_field2_id/id_parent' : ref(TABLE_1,'external_id_field') #indicate a self dependencies on openerp_field2_id 'state' : map_val('state_equivalent_field', mapping), # use get_state_map to make the mapping between the value of the field and the value of the state 'text_field' : concat('field_1', 'field_2', .., delimiter=':'), #concat the value of the list of field in one 'description_field' : ppconcat('field_1', 'field_2', .., delimiter='\n\t'), #same as above but with a prettier formatting 'field' : call(callable, arg1, arg2, ..), #call the function with all the value, the function should send the value : self.callable 'field' : callable 'field' : call(method, val('external_field') interface of method is self, val where val is the value of the field 'field' : const(value) #always set this field to value + any custom mapper that you will define } },
} """
return {}
def default_hook(self, val): """
this hook will be apply on each table that don't have hook here we define the identity hook """
return val
def _import_table(self, table): self.logger.info('Import table %s' % table) data = self.get_data(table) map = self.get_mapping()[table]['map'] hook = self.get_mapping()[table].get('hook', self.default_hook) model = self.get_mapping()[table]['model']
final_data = [] for val in data: res = hook(val) if res: final_data.append(res) return self._save_data(model, dict(map), final_data, table)
def _save_data(self, model, mapping, datas, table): """
@param model: the model of the object to import @param table : the external table where the data come from @param mapping : definition of the mapping @see: get_mapping @param datas : list of dictionnaries datas = [data_1, data_2, ..] data_i is a map external field_name => value and each data_i have a external id => in data_id['id'] """
self.logger.info(' Importing %s into %s' % (table, model)) if not datas: return (0, 'No data found') mapping['id'] = 'id_new' res = []
self_dependencies = [] for k in mapping.keys(): if '_parent' in k: self_dependencies.append((k[:-7], mapping.pop(k))) for data in datas: for k, field_name in self_dependencies: data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table)
data['id_new'] = self._generate_xml_id(self.get_external_id(data), table) fields, values = self._fields_mapp(data, mapping, table) res.append(values)
model_obj = self.obj.pool.get(model) if not model_obj: raise ValueError(_("%s is not a valid model name") % model) self.logger.info(_("fields imported : ") + str(fields)) (p, r, warning, s) = model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=False, context=self.context) self.logger.info('%s %s %s %s %s' % ("Done", p, r, warning, s)) for (field, field_name) in self_dependencies: self.logger.info('Import parent %s' % field) self._import_self_dependencies(model_obj, field, datas) return (len(res), warning)
def _import_self_dependencies(self, obj, parent_field, datas): """
@param parent_field: the name of the field that generate a self_dependencies, we call the object referenced in this field the parent of the object @param datas: a list of dictionnaries Dictionnaries need to contains id_new : the xml_id of the object field_new : the xml_id of the parent """
fields = ['id', parent_field] for data in datas: if data.get(parent_field): values = [data['id_new'], data[parent_field]] res = obj.import_data(self.cr, self.uid, fields, [values], mode='update', current_module=self.module_name, noupdate=False, context=self.context)
def _preprocess_mapping(self, mapping): """
Preprocess the mapping : after the preprocces, everything is callable in the val of the dictionary
use to allow syntaxical sugar like 'field': 'external_field' instead of 'field' : value('external_field') """
map = dict(mapping) for key, value in map.items(): if isinstance(value, basestring): map[key] = mapper.value(value) #set parent for instance of dbmapper elif isinstance(value, mapper.dbmapper): value.set_parent(self) return map
def _fields_mapp(self,dict_sugar, openerp_dict, table): """
call all the mapper and transform data to be compatible with import_data """
fields=[] data_lst = [] mapping = self._preprocess_mapping(openerp_dict) for key,val in mapping.items(): if key not in fields and dict_sugar: fields.append(key) value = val(dict(dict_sugar)) data_lst.append(value) return fields, data_lst
def _generate_xml_id(self, name, table): """
@param name: name of the object, has to be unique in for a given table @param table : table where the record we want generate come from @return: a unique xml id for record, the xml_id will be the same given the same table and same name To be used to avoid duplication of data that don't have ids """
sugar_instance = self.instance_name name = name.replace('.', '_').replace(',', '_') return sugar_instance + "_" + table + "_" + name
"""
Public interface of the framework those function can be use in the callable function defined in the mapping """
def xml_id_exist(self, table, external_id): """
Check if the external id exist in the openerp database in order to check if the id exist the table where it come from should be provide @return the xml_id generated if the external_id exist in the database or false """
if not external_id: return False
xml_id = self._generate_xml_id(external_id, table) id = self.obj.pool.get('ir.model.data').search(self.cr, self.uid, [('name', '=', xml_id), ('module', '=', self.module_name)]) return id and xml_id or False
def name_exist(self, table, name, model): """
Check if the object with the name exist in the openerp database in order to check if the id exist the table where it come from should be provide and the model of the object """
fields = ['name'] data = [name] return self.import_object(fields, data, model, table, name, [('name', '=', name)])
def get_mapped_id(self, table, external_id, context=None): """
@return return the databse id linked with the external_id """
if not external_id: return False
xml_id = self._generate_xml_id(external_id, table) return self.obj.pool.get('ir.model.data').get_object_reference(self.cr, self.uid, self.module_name, xml_id)[1]
def import_object_mapping(self, mapping, data, model, table, name, domain_search=False): """
same as import_objects but instead of two list fields and data, this method take a dictionnaries : external_field : value and the mapping similar to the one define in 'map' key @see import_object, get_mapping """
fields, datas = self._fields_mapp(data, mapping, table) return self.import_object(fields, datas, model, table, name, domain_search)
def import_object(self, fields, data, model, table, name, domain_search=False): """
This method will import an object in the openerp, usefull for field that is only a char in sugar and is an object in openerp use import_data that will take care to create/update or do nothing with the data this method return the xml_id
To be use, when you want to create an object or link if already exist use DO_NOT_LINK_DOMAIN to create always a new object @param fields: list of fields needed to create the object without id @param data: the list of the data, in the same order as the field ex : fields = ['firstname', 'lastname'] ; data = ['John', 'Mc donalds'] @param model: the openerp's model of the create/update object @param table: the table where data come from in sugarcrm, no need to fit the real name of openerp name, just need to be unique @param unique_name: the name of the object that we want to create/update get the id @param domain_search : the domain that should find the unique existing record
@return: the xml_id of the ressources """
domain_search = not domain_search and [('name', 'ilike', name)] or domain_search obj = self.obj.pool.get(model) if not obj: #if the model doesn't exist return False
xml_id = self._generate_xml_id(name, table) xml_ref = self.mapped_id_if_exist(model, domain_search, table, name) fields.append('id') data.append(xml_id) obj.import_data(self.cr, self.uid, fields, [data], mode='update', current_module=self.module_name, noupdate=True, context=self.context) return xml_ref or xml_id
def mapped_id_if_exist(self, model, domain, table, name): """
To be use when we want link with and existing object, if the object don't exist just ignore. @param domain : search domain to find existing record, should return a unique record @param xml_id: xml_id give to the mapping @param name: external_id or name of the object to create if there is no id @param table: the name of the table of the object to map @return : the xml_id if the record exist in the db, False otherwise """
obj = self.obj.pool.get(model) ids = obj.search(self.cr, self.uid, domain, context=self.context) if ids: xml_id = self._generate_xml_id(name, table) ir_model_data_obj = obj.pool.get('ir.model.data') id = ir_model_data_obj._update(self.cr, self.uid, model, self.module_name, {}, mode='update', xml_id=xml_id, noupdate=True, res_id=ids[0], context=self.context) return xml_id return False
def set_table_list(self, table_list): """
Set the list of table to import, this method should be call before run @param table_list: the list of external table to import ['Leads', 'Opportunity'] """
self.table_list = table_list
def launch_import(self): """
Import all data into openerp, this is the Entry point to launch the process of import
"""
self.data_started = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") error = False result = [] try: self.init_run() imported = set() #to invoid importing 2 times the sames modules for table in self.table_list: to_import = self.get_mapping()[table].get('import', True) if not table in imported: res = self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported) result.extend(res) if to_import: (position, warning) = self._import_table(table) result.append((table, position, warning)) imported.add(table) self.cr.commit()
except Exception, err: sh = StringIO.StringIO() traceback.print_exc(file=sh) error = sh.getvalue() self.logger.error(error)
self.date_ended = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def _resolve_dependencies(self, dep, imported): """
import dependencies recursively and avoid to import twice the same table """
result = [] for dependency in dep: if not dependency in imported: to_import = self.get_mapping()[dependency].get('import', True) res = self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported) result.extend(res) if to_import: r = self._import_table(dependency) (position, warning) = r result.append((dependency, position, warning)) imported.add(dependency) return result
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
|
