OCA
/
server-tools
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Add file import/export 

POC external_file_location integration in connector_flow
pull/516/head
Valentin Chemiere 10 years ago
committed by Florian da Costa
parent
28cf13adf9
commit
eb665583aa
22 changed files with 1290 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 59
      external_file_location/README.rst
  2. 27
      external_file_location/__init__.py
  3. 49
      external_file_location/__openerp__.py
  4. 26
      external_file_location/abstract_task.py
  5. 39
      external_file_location/attachment.py
  6. 99
      external_file_location/attachment_view.xml
  7. 18
      external_file_location/cron.xml
  8. 60
      external_file_location/helper.py
  9. 69
      external_file_location/location.py
  10. 69
      external_file_location/location_view.xml
  11. 12
      external_file_location/menu.xml
  12. 3
      external_file_location/security/ir.model.access.csv
  13. 99
      external_file_location/task.py
  14. 46
      external_file_location/task_view.xml
  15. 26
      external_file_location/tasks/__init__.py
  16. 140
      external_file_location/tasks/abstract_fs.py
  17. 69
      external_file_location/tasks/filestore.py
  18. 67
      external_file_location/tasks/ftp.py
  19. 73
      external_file_location/tasks/sftp.py
  20. 24
      external_file_location/tests/__init__.py
  21. 75
      external_file_location/tests/mock_server.py
  22. 141
      external_file_location/tests/test_sftp.py

59
external_file_location/README.rst
View File

@ -0,0 +1,59 @@
.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
:alt: License
External File Location
======================
This module was written to extend the functionality of ir.attachment to support remote communication and allow you to import/export file to a remote server
Installation
============
To install this module, you need to:
* fs python module
* Paramiko python module
Usage
=====
To use this module, you need to:
* Add a location with your server infos
* Create a task with your file info and remote communication method
* A cron task will trigger each task
For further information, please visit:
* https://www.odoo.com/forum/help-1
Known issues / Roadmap
======================
Credits
=======
* Joel Grand-Guillaume Camptocamp
* initOS <http://initos.com>
* Valentin CHEMIERE <valentin.chemiere@akretion.com>
Contributors
------------
* Sebastien BEAU <sebastian.beau@akretion.com>
Maintainer
----------
* Valentin CHEMIERE <valentin.chemiere@akretion.com>
.. image:: http://odoo-community.org/logo.png
:alt: Odoo Community Association
:target: http://odoo-community.org
This module is maintained by the OCA.
OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.
To contribute to this module, please visit http://odoo-community.org.

27
external_file_location/__init__.py
View File

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2014 Akretion (http://www.akretion.com).
# @author Sébastien BEAU <sebastien.beau@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from . import attachment
from . import location
from . import task
from . import tasks
from . import tests

49
external_file_location/__openerp__.py
View File

@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
{
'name': 'external_file_location',
'version': '0.0.1',
'author': 'Akretion',
'website': 'www.akretion.com',
'license': 'AGPL-3',
'category': 'Generic Modules',
'depends': [
'attachment_metadata',
],
'external_dependencies': {
'python': [
'fs',
'paramiko',
],
},
'data': [
'menu.xml',
'attachment_view.xml',
'location_view.xml',
'task_view.xml',
'cron.xml',
'security/ir.model.access.csv',
],
'installable': True,
'application': True,
}

26
external_file_location/abstract_task.py
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from base64 import b64encode
class AbstractTask(object):
_name = None
_key = None
_synchronize_type = None
_default_port = None
_hide_login = False
_hide_password = False
_hide_port = False
def create_file(self, filename, data):
ir_attachment_id = self.env['ir.attachment.metadata'].create({
'name': filename,
'datas': b64encode(data),
'datas_fname': filename,
'task_id': self.task and self.task.id or False,
'location_id': self.task and self.task.location_id.id or False,
'external_hash': self.ext_hash
})
return ir_attachment_id

39
external_file_location/attachment.py
View File

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from openerp import models, fields
class IrAttachmentMetadata(models.Model):
_inherit = 'ir.attachment.metadata'
sync_date = fields.Datetime()
state = fields.Selection([
('pending', 'Pending'),
('failed', 'Failed'),
('done', 'Done'),
], readonly=False, required=True, default='pending')
state_message = fields.Text()
task_id = fields.Many2one('external.file.task', string='Task')
location_id = fields.Many2one('external.file.location', string='Location',
related='task_id.location_id', store=True
)

99
external_file_location/attachment_view.xml
View File

@ -0,0 +1,99 @@
<?xml version="1.0" encoding="UTF-8"?>
<openerp>
<data>
<record id="view_attachment_improved_form" model="ir.ui.view">
<field name="model">ir.attachment.metadata</field>
<field name="inherit_id" ref="attachment_metadata.view_attachment_improved_form" />
<field name="arch" type="xml">
<field name="url" position="after">
<field name="sync_date"/>
<field name="state"/>
<field name="state_message"/>
<field name="task_id"/>
<field name="location_id"/>
</field>
</field>
</record>
<record id="view_external_attachment_tree" model="ir.ui.view">
<field name="model">ir.attachment.metadata</field>
<field name="arch" type="xml">
<tree string="Attachments" >
<field name="name"/>
<field name="datas_fname"/>
<field name="task_id"/>
<field name="location_id"/>
<field name="type"/>
<field name="create_date"/>
<field name="state"/>
</tree>
</field>
</record>
<record id="view_external_attachment_search" model="ir.ui.view">
<field name="model">ir.attachment.metadata</field>
<field name="arch" type="xml">
<search string="Attachments">
<field name="name" filter_domain="['|', ('name','ilike',self), ('datas_fname','ilike',self)]" string="Attachment"/>
<field name="create_date"/>
<filter icon="terp-stage"
string="URL"
domain="[('type','=','url')]"/>
<filter icon="terp-stock_align_left_24"
string="Binary"
domain="[('type','=','binary')]"/>
<separator/>
<filter name="my_documents_filter"
string="My Document(s)"
icon="terp-personal"
domain="[('create_uid','=',uid)]"
help="Filter on my documents"/>
<field name="create_uid"/>
<field name="type"/>
<filter string="Pending" domain="[('state', '=', 'pending')]"/>
<filter string="Failed" domain="[('state', '=', 'failed')]"/>
<filter string="Done" domain="[('state', '=', 'done')]"/>
<group expand="0" string="Group By">
<filter string="Owner" icon="terp-personal" domain="[]" context="{'group_by':'create_uid'}"/>
<filter string="Type" icon="terp-stock_symbol-selection" domain="[]" context="{'group_by':'type'}" groups="base.group_no_one"/>
<filter string="Company" icon="terp-gtk-home" domain="[]" context="{'group_by':'company_id'}" groups="base.group_multi_company"/>
<filter string="Creation Month" icon="terp-go-month" domain="[]" context="{'group_by':'create_date'}"/>
<filter string="State" domain="[]" context="{'group_by': 'state'}"/>
</group>
</search>
</field>
</record>
<record id="action_attachment" model="ir.actions.act_window">
<field name="name">Attachments</field>
<field name="type">ir.actions.act_window</field>
<field name="res_model">ir.attachment.metadata</field>
<field name="view_type">form</field>
<field name="view_mode">tree,form</field>
<field name="view_id" eval="False"/>
<!-- <field name="domain">[('task_id', '!=', False)]</field> -->
<field name="search_view_id" ref="view_external_attachment_search"/>
</record>
<record id="ir_attachment_view2" model="ir.actions.act_window.view">
<field eval="10" name="sequence"/>
<field name="view_mode">tree</field>
<field name="view_id" ref="view_external_attachment_tree"/>
<field name="act_window_id" ref="action_attachment"/>
</record>
<record id="ir_attachment_view3" model="ir.actions.act_window.view">
<field eval="10" name="sequence"/>
<field name="view_mode">form</field>
<field name="view_id" ref="view_attachment_improved_form"/>
<field name="act_window_id" ref="action_attachment"/>
</record>
<menuitem id="menu_ir_attachment"
parent="menu_file_exchange"
sequence="20"
action="action_attachment"/>
</data>
</openerp>

18
external_file_location/cron.xml
View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<openerp>
<data>
<record model="ir.cron" id="cronjob_run_exchange_tasks">
<field name='name'>Run file exchange tasks</field>
<field name='interval_number'>30</field>
<field name='interval_type'>minutes</field>
<field name="numbercall">-1</field>
<field name="active">True</field>
<field name="doall" eval="False" />
<field name="model">external.file.task</field>
<field name="function">_run</field>
<field name="args">([])</field>
</record>
</data>
</openerp>

60
external_file_location/helper.py
View File

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Author: Joel Grand-Guillaume
# Copyright 2011-2012 Camptocamp SA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
def itersubclasses(cls, _seen=None):
"""
itersubclasses(cls)
Generator over all subclasses of a given class, in depth first order.
>>> list(itersubclasses(int)) == [bool]
True
>>> class A(object): pass
>>> class B(A): pass
>>> class C(A): pass
>>> class D(B,C): pass
>>> class E(D): pass
>>>
>>> for cls in itersubclasses(A):
... print(cls.__name__)
B
D
E
C
>>> # get ALL (new-style) classes currently defined
>>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
['type', ...'tuple', ...]
"""
if not isinstance(cls, type):
raise TypeError('itersubclasses must be called with '
'new-style classes, not %.100r' % cls
)
if _seen is None:
_seen = set()
try:
subs = cls.__subclasses__()
except TypeError: # fails only when cls is type
subs = cls.__subclasses__(cls)
for sub in subs:
if sub not in _seen:
_seen.add(sub)
yield sub
for sub in itersubclasses(sub, _seen):
yield sub

69
external_file_location/location.py
View File

@ -0,0 +1,69 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from openerp import models, fields, api
from .abstract_task import AbstractTask
from .helper import itersubclasses
class Location(models.Model):
_name = 'external.file.location'
_description = 'Description'
name = fields.Char(string='Name', required=True)
protocol = fields.Selection(selection='_get_protocol', required=True)
address = fields.Char(string='Address', required=True)
port = fields.Integer()
login = fields.Char()
password = fields.Char()
task_ids = fields.One2many('external.file.task', 'location_id')
hide_login = fields.Boolean()
hide_password = fields.Boolean()
hide_port = fields.Boolean()
def _get_protocol(self):
res = []
for cls in itersubclasses(AbstractTask):
if not cls._synchronize_type:
cls_info = (cls._key, cls._name)
res.append(cls_info)
elif not cls._synchronize_type and cls._key and cls._name:
pass
return res
@api.onchange('protocol')
def onchange_protocol(self):
for cls in itersubclasses(AbstractTask):
if cls._key == self.protocol:
self.port = cls._default_port
if cls._hide_login:
self.hide_login = True
else:
self.hide_login = False
if cls._hide_password:
self.hide_password = True
else:
self.hide_password = False
if cls._hide_port:
self.hide_port = True
else:
self.hide_port = False

69
external_file_location/location_view.xml
View File

@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<openerp>
<data>
<record id="view_location_form" model="ir.ui.view">
<field name="model">external.file.location</field>
<field name="arch" type="xml">
<form string="Location" version="7.0">
<sheet>
<group col="4">
<div class="oe_title" style="width: 390px;" colspan="4">
<label class="oe_edit_only" for="name" string="Name"/>
<h1><field name="name" class="oe_inline"/></h1>
</div>
<newline/>
<field name="protocol" colspan="2"/>
<newline/>
<field name="address" colspan="2"/>
<field name="port" colspan="2" attrs="{'invisible': [('hide_port', '=', True)], 'required': [('hide_port', '=', False)]}"/>
<field name="login" colspan="2" attrs="{'invisible': [('hide_login', '=', True)], 'required': [('hide_login', '=', False)]}"/>
<field name="password" password="1" colspan="2" attrs="{'invisible': [('hide_password', '=', True)]}"/>
<separator string="Tasks" colspan="4"/>
<field name="task_ids" colspan="4" nolabel="1" context="{'hide_location': True, 'protocol': protocol}">
<tree>
<field name="name"/>
<field name="name"/>
<field name="method"/>
<field name="filename"/>
<field name="filepath"/>
<button name="run" type="object" string="Run" icon="gtk-execute"/>
</tree>
</field>
<field name="hide_login" invisible="1"/>
<field name="hide_password" invisible="1"/>
<field name="hide_port" invisible="1"/>
</group>
</sheet>
</form>
</field>
</record>
<record id="view_location_tree" model="ir.ui.view">
<field name="model">external.file.location</field>
<field name="arch" type="xml">
<tree string="Location">
<field name="name" select="1"/>
<field name="protocol"/>
<field name="address"/>
<field name="login"/>
</tree>
</field>
</record>
<record id="action_location" model="ir.actions.act_window">
<field name="name">Locations</field>
<field name="type">ir.actions.act_window</field>
<field name="res_model">external.file.location</field>
<field name="view_type">form</field>
<field name="view_id" eval="False"/>
</record>
<menuitem id="menu_ir_location"
parent="menu_file_exchange"
sequence="20"
action="action_location"/>
</data>
</openerp>

12
external_file_location/menu.xml
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<openerp>
<data>
<menuitem id="menu_file_exchange"
parent="base.menu_administration"
sequence="20"
name="File exchange"
/>
</data>
</openerp>

3
external_file_location/security/ir.model.access.csv
View File

@ -0,0 +1,3 @@
id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
access_external_file_location_user,external.file.location.user,model_external_file_location,base.group_user,1,0,0,0
access_external_file_task_user,external.file.task.user,model_external_file_task,base.group_user,1,0,0,0

99
external_file_location/task.py
View File

@ -0,0 +1,99 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from openerp import models, fields, api
from .helper import itersubclasses
from .abstract_task import AbstractTask
class Task(models.Model):
_name = 'external.file.task'
_description = 'Description'
name = fields.Char(required=True)
method = fields.Selection(selection='_get_method', required=True,
help='procotol and trasmitting info')
method_type = fields.Char()
filename = fields.Char(help='File name which is imported')
filepath = fields.Char(help='Path to imported file')
location_id = fields.Many2one('external.file.location', string='Location',
required=True)
attachment_ids = fields.One2many('ir.attachment.metadata', 'task_id',
string='Attachment')
move_path = fields.Char(string='Move path',
help='Imported File will be moved to this path')
md5_check = fields.Boolean(help='Control file integrity after import with'
' a md5 file')
after_import = fields.Selection(selection='_get_action',
help='Action after import a file')
def _get_action(self):
return [('move', 'Move'), ('delete', 'Delete')]
def _get_method(self):
res = []
for cls in itersubclasses(AbstractTask):
if cls._synchronize_type \
and ('protocol' not in self._context
or cls._key == self._context['protocol']):
cls_info = (cls._key + '_' + cls._synchronize_type,
cls._name + ' ' + cls._synchronize_type)
res.append(cls_info)
return res
@api.onchange('method')
def onchange_method(self):
if self.method:
if 'import' in self.method:
self.method_type = 'import'
elif 'export' in self.method:
self.method_type = 'export'
@api.model
def _run(self, domain=None):
if not domain:
domain = []
tasks = self.env['external.file.task'].search(domain)
tasks.run()
@api.one
def run(self):
for cls in itersubclasses(AbstractTask):
if cls._synchronize_type and \
cls._key + '_' + cls._synchronize_type == self.method:
method_class = cls
config = {
'host': self.location_id.address,
'user': self.location_id.login,
'pwd': self.location_id.password,
'port': self.location_id.port,
'allow_dir_creation': False,
'file_name': self.filename,
'path': self.filepath,
'attachment_ids': self.attachment_ids,
'task': self,
'move_path': self.move_path,
'after_import': self.after_import,
'md5_check': self.md5_check,
}
conn = method_class(self.env, config)
conn.run()

46
external_file_location/task_view.xml
View File

@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<openerp>
<data>
<record id="view_task_form" model="ir.ui.view">
<field name="model">external.file.task</field>
<field name="arch" type="xml">
<form string="Tasks" version="7.0">
<header>
<button name="run" type="object" string="Run" icon="gtk-execute"/>
</header>
<sheet>
<field name="method_type" invisible="1"/>
<group col="4">
<div class="oe_title" style="width: 390px;" colspan="4">
<label class="oe_edit_only" for="name" string="Name"/>
<h1><field name="name" class="oe_inline"/></h1>
</div>
<field name="method" colspan="2"/>
<span colspan="2"/>
<field name="filename" colspan="4" attrs="{'invisible':[('method_type','!=','import')], 'required':[('method_type', '=', 'import')]}"/>
<field name="filepath" colspan="4" />
<field name="after_import" colspan="2" attrs="{'invisible':[('method_type','!=','import')]}"/>
<field name="move_path" colspan="2" attrs="{'invisible':['|', ('after_import','!=','move'), ('method_type','!=','import')]}"/>
<field name="md5_check" colspan="2" attrs="{'invisible':[('method_type','!=','import')]}"/>
</group>
</sheet>
</form>
</field>
</record>
<record id="view_task_tree" model="ir.ui.view">
<field name="model">external.file.task</field>
<field name="arch" type="xml">
<tree string="Tasks" >
<field name="name" select="1"/>
<field name="method"/>
<field name="filename"/>
<field name="filepath"/>
<button name="run" type="object" string="Run" icon="gtk-execute"/>
</tree>
</field>
</record>
</data>
</openerp>

26
external_file_location/tasks/__init__.py
View File

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from . import abstract_fs
from . import ftp
from . import sftp
from . import filestore

140
external_file_location/tasks/abstract_fs.py
View File

@ -0,0 +1,140 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from ..abstract_task import AbstractTask
import logging
import os
_logger = logging.getLogger(__name__)
class AbstractFSTask(AbstractTask):
_name = None
_key = None
_synchronize_type = None
_default_port = None
def __init__(self, env, config):
self.env = env
self.host = config.get('host', '')
self.user = config.get('user', '')
self.pwd = config.get('pwd', '')
self.port = config.get('port', '')
self.allow_dir_creation = config.get('allow_dir_creation', '')
self.file_name = config.get('file_name', '')
self.path = config.get('path') or '.'
self.move_path = config.get('move_path', '')
self.after_import = config.get('after_import', False)
self.attachment_ids = config.get('attachment_ids', False)
self.task = config.get('task', False)
self.ext_hash = False
self.md5_check = config.get('md5_check', False)
def _handle_new_source(self, fs_conn, download_directory, file_name,
move_directory):
"""open and read given file into create_file method,
move file if move_directory is given"""
with fs_conn.open(self._source_name(download_directory, file_name),
"rb") as fileobj:
data = fileobj.read()
return self.create_file(file_name, data)
def _source_name(self, download_directory, file_name):
"""helper to get the full name"""
return os.path.join(download_directory, file_name)
def _move_file(self, fs_conn, source, target):
"""Moves a file on the server"""
_logger.info('Moving file %s %s' % (source, target))
fs_conn.rename(source, target)
if self.md5_check:
fs_conn.rename(source + '.md5', target + '.md5')
def _delete_file(self, fs_conn, source):
"""Deletes a file from the server"""
_logger.info('Deleting file %s' % source)
fs_conn.remove(source)
if self.md5_check:
fs_conn.remove(source + '.md5')
def _get_hash(self, file_name, fs_conn):
hash_file_name = file_name + '.md5'
with fs_conn.open(hash_file_name, 'rb') as f:
return f.read().rstrip('\r\n')
def _get_files(self, conn, path):
process_files = []
files_list = conn.listdir(path)
for file in files_list:
if file == self.file_name:
source_name = self._source_name(self.path, self.file_name)
process_files.append((file, source_name))
return process_files
def _process_file(self, conn, file_to_process):
if self.md5_check:
self.ext_hash = self._get_hash(file_to_process[1], conn)
att_id = self._handle_new_source(
conn,
self.path,
self.file_name,
self.move_path)
# Move/delete files only after all files have been processed.
if self.after_import == 'delete':
self._delete_file(conn, file_to_process[1])
elif self.after_import == 'move':
if not conn.exists(self.move_path):
conn.makedir(self.move_path)
self._move_file(
conn,
file_to_process[1],
self._source_name(self.move_path, file_to_process[0]))
return att_id
def _handle_existing_target(self, fs_conn, target_name, filedata):
raise Exception("%s already exists" % target_name)
def _handle_new_target(self, fs_conn, target_name, filedata):
try:
with fs_conn.open(target_name, mode='wb') as fileobj:
fileobj.write(filedata)
_logger.info('wrote %s, size %d', target_name, len(filedata))
self.attachment_id.state = 'done'
self.attachment_id.state_message = ''
except IOError:
self.attachment_id.state = 'failed'
self.attachment_id.state_message = (
'The directory doesn\'t exist or had insufficient rights')
def _target_name(self, fs_conn, upload_directory, filename):
return os.path.join(upload_directory, filename)
def _upload_file(self, conn, host, port, user, pwd,
path, filename, filedata):
upload_directory = path or '.'
target_name = self._target_name(conn,
upload_directory,
filename)
if conn.isfile(target_name):
self._handle_existing_target(conn, target_name, filedata)
else:
self._handle_new_target(conn, target_name, filedata)

69
external_file_location/tasks/filestore.py
View File

@ -0,0 +1,69 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from .abstract_fs import AbstractFSTask
from base64 import b64decode
from fs import osfs
import logging
_logger = logging.getLogger(__name__)
class FileStoreTask(AbstractFSTask):
_key = 'filestore'
_name = 'File Store'
_synchronize_type = None
_default_port = None
_hide_login = True
_hide_password = True
_hide_port = True
class FileStoreImportTask(FileStoreTask):
_synchronize_type = 'import'
def run(self):
att_ids = []
with osfs.OSFS(self.host) as fs_conn:
files_to_process = self._get_files(fs_conn, self.path)
for file_to_process in files_to_process:
att_ids.append(self._process_file(fs_conn, file_to_process))
return att_ids
class FileStoreExportTask(FileStoreTask):
_synchronize_type = 'export'
def run(self, async=True):
for attachment in self.attachment_ids:
if attachment.state in ('pending', 'failed'):
self.attachment_id = attachment
with osfs.OSFS(self.host) as fs_conn:
self._upload_file(fs_conn,
self.host,
self.port,
self.user,
self.pwd,
self.path,
attachment.datas_fname,
b64decode(attachment.datas))

67
external_file_location/tasks/ftp.py
View File

@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from .abstract_fs import AbstractFSTask
from base64 import b64decode
from fs import ftpfs
import logging
_logger = logging.getLogger(__name__)
class FtpTask(AbstractFSTask):
_key = 'ftp'
_name = 'FTP'
_synchronize_type = None
_default_port = 21
_hide_login = False
_hide_password = False
_hide_port = False
class FtpImportTask(FtpTask):
_synchronize_type = 'import'
def run(self):
att_ids = []
with ftpfs.FTPFS(self.host, self.user, self.pwd,
port=self.port) as ftp_conn:
files_to_process = self._get_files(ftp_conn, self.path)
for file_to_process in files_to_process:
att_ids.append(self._process_file(ftp_conn, file_to_process))
return att_ids
class FtpExportTask(FtpTask):
_synchronize_type = 'export'
def run(self, async=True):
import ipdb; ipdb.set_trace()
for attachment in self.attachment_ids:
if attachment.state in ('pending', 'failed'):
self.attachment_id = attachment
with ftpfs.FTPFS(self.host, self.user, self.pwd,
port=self.port) as ftp_conn:
self._upload_file(ftp_conn, self.host, self.port,
self.user, self.pwd, self.path,
attachment.datas_fname,
b64decode(attachment.datas))

73
external_file_location/tasks/sftp.py
View File

@ -0,0 +1,73 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from .abstract_fs import AbstractFSTask
from base64 import b64decode
from fs import sftpfs
import logging
_logger = logging.getLogger(__name__)
class SftpTask(AbstractFSTask):
_key = 'sftp'
_name = 'SFTP'
_synchronize_type = None
_default_port = 22
_hide_login = False
_hide_password = False
_hide_port = False
class SftpImportTask(SftpTask):
_synchronize_type = 'import'
def run(self):
connection_string = "{}:{}".format(self.host, self.port)
root = "/home/{}".format(self.user)
att_ids = []
with sftpfs.SFTPFS(connection=connection_string,
root_path=root,
username=self.user,
password=self.pwd) as sftp_conn:
files_to_process = self._get_files(sftp_conn, self.path)
for file_to_process in files_to_process:
att_ids.append(self._process_file(sftp_conn, file_to_process))
return att_ids
class SftpExportTask(SftpTask):
_synchronize_type = 'export'
def run(self, async=True):
for attachment in self.attachment_ids:
if attachment.state in ('pending', 'failed'):
self.attachment_id = attachment
connection_string = "{}:{}".format(self.host, self.port)
with sftpfs.SFTPFS(connection=connection_string,
username=self.user,
password=self.pwd) as sftp_conn:
datas = b64decode(attachment.datas)
self._upload_file(sftp_conn, self.host, self.port,
self.user, self.pwd, self.path,
attachment.datas_fname, datas)

24
external_file_location/tests/__init__.py
View File

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from . import mock_server
from . import test_sftp

75
external_file_location/tests/mock_server.py
View File

@ -0,0 +1,75 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import mock
from contextlib import contextmanager
from collections import defaultdict
class MultiResponse(dict):
pass
class ConnMock(object):
def __init__(self, response):
self.response = response
self._calls = []
self.call_count = defaultdict(int)
def __getattribute__(self, method):
if method not in ('_calls', 'response', 'call_count'):
def callable(*args, **kwargs):
self._calls.append({
'method': method,
'args': args,
'kwargs': kwargs,
})
call = self.response[method]
if isinstance(call, MultiResponse):
call = call[self.call_count[method]]
self.call_count[method] += 1
return call
return callable
else:
return super(ConnMock, self).__getattribute__(method)
def __call__(self, *args, **kwargs):
return self
def __enter__(self, *args, **kwargs):
return self
def __exit__(self, *args, **kwargs):
pass
def __repr__(self, *args, **kwargs):
return self
def __getitem__(self, key):
return
@contextmanager
def server_mock(response):
with mock.patch('fs.sftpfs.SFTPFS', ConnMock(response)) as SFTPFS:
yield SFTPFS._calls

141
external_file_location/tests/test_sftp.py
View File

@ -0,0 +1,141 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import openerp.tests.common as common
from ..tasks.sftp import SftpImportTask
from ..tasks.sftp import SftpExportTask
from .mock_server import (server_mock)
from .mock_server import MultiResponse
from StringIO import StringIO
from base64 import b64decode
import hashlib
class ContextualStringIO(StringIO):
"""
snippet from http://bit.ly/1HfH6uW (stackoverflow)
"""
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
return False
class TestNewSource(common.TransactionCase):
def setUp(self):
super(TestNewSource, self).setUp()
self.test_file = ContextualStringIO()
self.test_file.write('import')
self.test_file.seek(0)
self.config = \
{'file_name': 'testfile',
'user': 'test',
'password': 'test',
'host': 'test',
'port': 22,
'attachment_ids': self.env['ir.attachment.metadata'].search([])
}
def test_00_sftp_import(self):
with server_mock(
{'exists': True,
'makedir': True,
'open': self.test_file,
'listdir': ['testfile']
}):
task = SftpImportTask(self.env, self.config)
task.run()
search_file = self.env['ir.attachment.metadata'].search(
(('name', '=', 'testfile'),))
self.assertEqual(len(search_file), 1)
self.assertEqual(b64decode(search_file[0].datas), 'import')
def test_01_sftp_export(self):
with server_mock(
{'isfile': False,
'open': self.test_file,
}) as FakeSFTP:
task = SftpExportTask(self.env, self.config)
task.run()
if FakeSFTP:
self.assertEqual('open', FakeSFTP[-1]['method'])
def test_02_sftp_import_delete(self):
with server_mock(
{'exists': True,
'makedir': True,
'open': self.test_file,
'listdir': ['testfile'],
'remove': True
}) as FakeSFTP:
self.config.update({'after_import': 'delete'})
task = SftpImportTask(self.env, self.config)
task.run()
search_file = self.env['ir.attachment.metadata'].search(
(('name', '=', 'testfile'),))
self.assertEqual(len(search_file), 1)
self.assertEqual(b64decode(search_file[0].datas), 'import')
self.assertEqual('remove', FakeSFTP[-1]['method'])
def test_03_sftp_import_move(self):
with server_mock(
{'exists': True,
'makedir': True,
'open': self.test_file,
'listdir': ['testfile'],
'rename': True
}) as FakeSFTP:
self.config.update({'after_import': 'move', 'move_path': '/home'})
task = SftpImportTask(self.env, self.config)
task.run()
search_file = self.env['ir.attachment.metadata'].search(
(('name', '=', 'testfile'),))
self.assertEqual(len(search_file), 1)
self.assertEqual(b64decode(search_file[0].datas), 'import')
self.assertEqual('rename', FakeSFTP[-1]['method'])
def test_04_sftp_import_md5(self):
md5_file = ContextualStringIO()
md5_file.write(hashlib.md5('import').hexdigest())
md5_file.seek(0)
with server_mock(
{'exists': True,
'makedir': True,
'open': MultiResponse({
1: self.test_file,
0: md5_file
}),
'listdir': ['testfile', 'testfile.md5'],
}) as FakeSFTP:
self.config.update({'md5_check': True})
task = SftpImportTask(self.env, self.config)
task.run()
search_file = self.env['ir.attachment.metadata'].search(
(('name', '=', 'testfile'),))
self.assertEqual(len(search_file), 1)
self.assertEqual(b64decode(search_file[0].datas), 'import')
self.assertEqual('open', FakeSFTP[-1]['method'])
self.assertEqual('open', FakeSFTP[1]['method'])
self.assertEqual(('./testfile.md5', 'rb'), FakeSFTP[1]['args'])
Write Preview
Loading…
Cancel
Save