Add file import/export

POC external_file_location integration in connector_flow

external_file_location/README.rst

@@ -0,0 +1,59 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :alt: License
+
+External File Location
+======================
+
+This module was written to extend the functionality of ir.attachment to support remote communication and allow you to import/export file to a remote server
+
+Installation
+============
+
+To install this module, you need to:
+
+* fs python module
+* Paramiko python module
+
+Usage
+=====
+
+To use this module, you need to:
+
+* Add a location with your server infos
+* Create a task with your file info and remote communication method
+* A cron task will trigger each task
+
+For further information, please visit:
+
+* https://www.odoo.com/forum/help-1
+
+Known issues / Roadmap
+======================
+
+
+Credits
+=======
+
+* Joel Grand-Guillaume Camptocamp
+* initOS <http://initos.com>
+* Valentin CHEMIERE <valentin.chemiere@akretion.com>
+
+Contributors
+------------
+
+* Sebastien BEAU <sebastian.beau@akretion.com>
+
+Maintainer
+----------
+
+* Valentin CHEMIERE <valentin.chemiere@akretion.com>
+
+.. image:: http://odoo-community.org/logo.png
+    :alt: Odoo Community Association
+    :target: http://odoo-community.org
+
+This module is maintained by the OCA.
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.
+
+To contribute to this module, please visit http://odoo-community.org.

external_file_location/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2014 Akretion (http://www.akretion.com).
+#   @author Sébastien BEAU <sebastien.beau@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from . import attachment
+from . import location
+from . import task
+from . import tasks
+from . import tests

external_file_location/__openerp__.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+{
+    'name': 'external_file_location',
+    'version': '0.0.1',
+    'author': 'Akretion',
+    'website': 'www.akretion.com',
+    'license': 'AGPL-3',
+    'category': 'Generic Modules',
+    'depends': [
+        'attachment_metadata',
+    ],
+    'external_dependencies': {
+        'python': [
+            'fs',
+            'paramiko',
+        ],
+    },
+    'data': [
+        'menu.xml',
+        'attachment_view.xml',
+        'location_view.xml',
+        'task_view.xml',
+        'cron.xml',
+        'security/ir.model.access.csv',
+    ],
+    'installable': True,
+    'application': True,
+    }

external_file_location/abstract_task.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from base64 import b64encode
+
+
+class AbstractTask(object):
+
+    _name = None
+    _key = None
+    _synchronize_type = None
+    _default_port = None
+    _hide_login = False
+    _hide_password = False
+    _hide_port = False
+
+    def create_file(self, filename, data):
+        ir_attachment_id = self.env['ir.attachment.metadata'].create({
+            'name': filename,
+            'datas': b64encode(data),
+            'datas_fname': filename,
+            'task_id': self.task and self.task.id or False,
+            'location_id': self.task and self.task.location_id.id or False,
+            'external_hash': self.ext_hash
+            })
+        return ir_attachment_id

external_file_location/attachment.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from openerp import models, fields
+
+
+class IrAttachmentMetadata(models.Model):
+    _inherit = 'ir.attachment.metadata'
+
+    sync_date = fields.Datetime()
+    state = fields.Selection([
+        ('pending', 'Pending'),
+        ('failed', 'Failed'),
+        ('done', 'Done'),
+        ], readonly=False, required=True, default='pending')
+    state_message = fields.Text()
+    task_id = fields.Many2one('external.file.task', string='Task')
+    location_id = fields.Many2one('external.file.location', string='Location',
+                                  related='task_id.location_id', store=True
+                                  )

external_file_location/attachment_view.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+
+        <record id="view_attachment_improved_form" model="ir.ui.view">
+            <field name="model">ir.attachment.metadata</field>
+            <field name="inherit_id" ref="attachment_metadata.view_attachment_improved_form" />
+            <field name="arch" type="xml">
+            <field name="url" position="after">
+                <field name="sync_date"/>
+                <field name="state"/>
+                <field name="state_message"/>
+                <field name="task_id"/>
+                <field name="location_id"/>
+                </field>
+            </field>
+        </record>
+
+        <record id="view_external_attachment_tree" model="ir.ui.view">
+            <field name="model">ir.attachment.metadata</field>
+            <field name="arch" type="xml">
+                <tree string="Attachments" >
+                    <field name="name"/>
+                    <field name="datas_fname"/>
+                    <field name="task_id"/>
+                    <field name="location_id"/>
+                    <field name="type"/>
+                    <field name="create_date"/>
+                    <field name="state"/>
+                </tree>
+            </field>
+        </record>
+
+        <record id="view_external_attachment_search" model="ir.ui.view">
+            <field name="model">ir.attachment.metadata</field>
+            <field name="arch" type="xml">
+                <search string="Attachments">
+                    <field name="name" filter_domain="['|', ('name','ilike',self), ('datas_fname','ilike',self)]" string="Attachment"/>
+                    <field name="create_date"/>
+                    <filter icon="terp-stage"
+                        string="URL"
+                        domain="[('type','=','url')]"/>
+                    <filter icon="terp-stock_align_left_24"
+                        string="Binary"
+                        domain="[('type','=','binary')]"/>
+                    <separator/>
+                    <filter name="my_documents_filter" 
+                        string="My Document(s)"
+                        icon="terp-personal"
+                        domain="[('create_uid','=',uid)]"
+                        help="Filter on my documents"/>
+                    <field name="create_uid"/>
+                    <field name="type"/>
+                    <filter string="Pending" domain="[('state', '=', 'pending')]"/>
+                    <filter string="Failed" domain="[('state', '=', 'failed')]"/>
+                    <filter string="Done" domain="[('state', '=', 'done')]"/>
+                    <group expand="0" string="Group By">
+                        <filter string="Owner" icon="terp-personal" domain="[]" context="{'group_by':'create_uid'}"/>
+                        <filter string="Type" icon="terp-stock_symbol-selection" domain="[]" context="{'group_by':'type'}" groups="base.group_no_one"/>
+                        <filter string="Company" icon="terp-gtk-home" domain="[]" context="{'group_by':'company_id'}" groups="base.group_multi_company"/>
+                        <filter string="Creation Month" icon="terp-go-month" domain="[]" context="{'group_by':'create_date'}"/>
+                        <filter string="State" domain="[]" context="{'group_by': 'state'}"/>
+                    </group>
+                </search>
+            </field>
+        </record>
+
+        <record id="action_attachment" model="ir.actions.act_window">
+            <field name="name">Attachments</field>
+            <field name="type">ir.actions.act_window</field>
+            <field name="res_model">ir.attachment.metadata</field>
+            <field name="view_type">form</field>
+            <field name="view_mode">tree,form</field>
+            <field name="view_id" eval="False"/>
+            <!-- <field name="domain">[('task_id', '!=', False)]</field> -->
+            <field name="search_view_id" ref="view_external_attachment_search"/>
+        </record>
+
+        <record id="ir_attachment_view2" model="ir.actions.act_window.view">
+            <field eval="10" name="sequence"/>
+            <field name="view_mode">tree</field>
+            <field name="view_id" ref="view_external_attachment_tree"/>
+            <field name="act_window_id" ref="action_attachment"/>
+        </record>
+
+        <record id="ir_attachment_view3" model="ir.actions.act_window.view">
+            <field eval="10" name="sequence"/>
+            <field name="view_mode">form</field>
+            <field name="view_id" ref="view_attachment_improved_form"/>
+            <field name="act_window_id" ref="action_attachment"/>
+        </record>
+
+        <menuitem id="menu_ir_attachment" 
+            parent="menu_file_exchange"
+            sequence="20"
+            action="action_attachment"/>
+
+    </data>
+</openerp>

external_file_location/cron.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+        
+        <record model="ir.cron" id="cronjob_run_exchange_tasks">
+            <field name='name'>Run file exchange tasks</field>
+            <field name='interval_number'>30</field>
+            <field name='interval_type'>minutes</field>
+            <field name="numbercall">-1</field>
+            <field name="active">True</field>
+            <field name="doall" eval="False" />
+            <field name="model">external.file.task</field>
+            <field name="function">_run</field>
+            <field name="args">([])</field>
+        </record>
+
+    </data>
+</openerp>

external_file_location/helper.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Joel Grand-Guillaume
+# Copyright 2011-2012 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+
+def itersubclasses(cls, _seen=None):
+    """
+    itersubclasses(cls)
+    Generator over all subclasses of a given class, in depth first order.
+    >>> list(itersubclasses(int)) == [bool]
+    True
+    >>> class A(object): pass
+    >>> class B(A): pass
+    >>> class C(A): pass
+    >>> class D(B,C): pass
+    >>> class E(D): pass
+    >>>
+    >>> for cls in itersubclasses(A):
+    ... print(cls.__name__)
+    B
+    D
+    E
+    C
+    >>> # get ALL (new-style) classes currently defined
+    >>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
+    ['type', ...'tuple', ...]
+    """
+    if not isinstance(cls, type):
+        raise TypeError('itersubclasses must be called with '
+                        'new-style classes, not %.100r' % cls
+                        )
+    if _seen is None:
+        _seen = set()
+    try:
+        subs = cls.__subclasses__()
+    except TypeError:  # fails only when cls is type
+        subs = cls.__subclasses__(cls)
+    for sub in subs:
+        if sub not in _seen:
+            _seen.add(sub)
+            yield sub
+            for sub in itersubclasses(sub, _seen):
+                yield sub

external_file_location/location.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from openerp import models, fields, api
+from .abstract_task import AbstractTask
+from .helper import itersubclasses
+
+
+class Location(models.Model):
+    _name = 'external.file.location'
+    _description = 'Description'
+
+    name = fields.Char(string='Name', required=True)
+    protocol = fields.Selection(selection='_get_protocol', required=True)
+    address = fields.Char(string='Address', required=True)
+    port = fields.Integer()
+    login = fields.Char()
+    password = fields.Char()
+    task_ids = fields.One2many('external.file.task', 'location_id')
+    hide_login = fields.Boolean()
+    hide_password = fields.Boolean()
+    hide_port = fields.Boolean()
+
+    def _get_protocol(self):
+        res = []
+        for cls in itersubclasses(AbstractTask):
+            if not cls._synchronize_type:
+                cls_info = (cls._key, cls._name)
+                res.append(cls_info)
+            elif not cls._synchronize_type and cls._key and cls._name:
+                pass
+        return res
+
+    @api.onchange('protocol')
+    def onchange_protocol(self):
+        for cls in itersubclasses(AbstractTask):
+            if cls._key == self.protocol:
+                self.port = cls._default_port
+                if cls._hide_login:
+                    self.hide_login = True
+                else:
+                    self.hide_login = False
+                if cls._hide_password:
+                    self.hide_password = True
+                else:
+                    self.hide_password = False
+                if cls._hide_port:
+                    self.hide_port = True
+                else:
+                    self.hide_port = False

external_file_location/location_view.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+
+        <record id="view_location_form" model="ir.ui.view">
+            <field name="model">external.file.location</field>
+            <field name="arch" type="xml">
+                <form string="Location" version="7.0">
+                    <sheet>
+                        <group col="4">
+                            <div class="oe_title" style="width: 390px;" colspan="4">
+                                <label class="oe_edit_only" for="name" string="Name"/>
+                                <h1><field name="name" class="oe_inline"/></h1>
+                            </div>
+                            <newline/>
+                            <field name="protocol" colspan="2"/>
+                            <newline/>
+                            <field name="address" colspan="2"/>
+                            <field name="port" colspan="2" attrs="{'invisible': [('hide_port', '=', True)], 'required': [('hide_port', '=', False)]}"/>
+                            <field name="login" colspan="2" attrs="{'invisible': [('hide_login', '=', True)], 'required': [('hide_login', '=', False)]}"/>
+                            <field name="password" password="1" colspan="2" attrs="{'invisible': [('hide_password', '=', True)]}"/>
+                            <separator string="Tasks" colspan="4"/>
+                            <field name="task_ids" colspan="4" nolabel="1" context="{'hide_location': True, 'protocol': protocol}">
+                                <tree>
+                                    <field name="name"/>
+                                    <field name="name"/>
+                                    <field name="method"/>
+                                    <field name="filename"/>
+                                    <field name="filepath"/>
+                                    <button name="run" type="object" string="Run" icon="gtk-execute"/>
+                                </tree>
+                            </field>
+                            <field name="hide_login" invisible="1"/>
+                            <field name="hide_password" invisible="1"/>
+                            <field name="hide_port" invisible="1"/>
+                        </group>
+                    </sheet>
+                </form>
+            </field>
+        </record>
+
+        <record id="view_location_tree" model="ir.ui.view">
+            <field name="model">external.file.location</field>
+            <field name="arch" type="xml">
+                <tree string="Location">
+                    <field name="name" select="1"/>
+                    <field name="protocol"/>
+                    <field name="address"/>
+                    <field name="login"/>
+                </tree>
+            </field>
+        </record>
+
+        <record id="action_location" model="ir.actions.act_window">
+            <field name="name">Locations</field>
+            <field name="type">ir.actions.act_window</field>
+            <field name="res_model">external.file.location</field>
+            <field name="view_type">form</field>
+            <field name="view_id" eval="False"/>
+        </record>
+
+        <menuitem id="menu_ir_location" 
+            parent="menu_file_exchange"
+            sequence="20"
+            action="action_location"/>
+
+    </data>
+</openerp>
+ 

external_file_location/menu.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+        
+        <menuitem id="menu_file_exchange" 
+            parent="base.menu_administration"
+            sequence="20"
+            name="File exchange"
+            />
+
+    </data>
+</openerp>

external_file_location/security/ir.model.access.csv

@@ -0,0 +1,3 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_external_file_location_user,external.file.location.user,model_external_file_location,base.group_user,1,0,0,0
+access_external_file_task_user,external.file.task.user,model_external_file_task,base.group_user,1,0,0,0

external_file_location/task.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from openerp import models, fields, api
+from .helper import itersubclasses
+from .abstract_task import AbstractTask
+
+
+class Task(models.Model):
+    _name = 'external.file.task'
+    _description = 'Description'
+
+    name = fields.Char(required=True)
+    method = fields.Selection(selection='_get_method', required=True,
+                              help='procotol and trasmitting info')
+    method_type = fields.Char()
+    filename = fields.Char(help='File name which is imported')
+    filepath = fields.Char(help='Path to imported file')
+    location_id = fields.Many2one('external.file.location', string='Location',
+                                  required=True)
+    attachment_ids = fields.One2many('ir.attachment.metadata', 'task_id',
+                                     string='Attachment')
+    move_path = fields.Char(string='Move path',
+                            help='Imported File will be moved to this path')
+    md5_check = fields.Boolean(help='Control file integrity after import with'
+                               ' a md5 file')
+    after_import = fields.Selection(selection='_get_action',
+                                    help='Action after import a file')
+
+    def _get_action(self):
+        return [('move', 'Move'), ('delete', 'Delete')]
+
+    def _get_method(self):
+        res = []
+        for cls in itersubclasses(AbstractTask):
+            if cls._synchronize_type \
+                and ('protocol' not in self._context
+                     or cls._key == self._context['protocol']):
+                cls_info = (cls._key + '_' + cls._synchronize_type,
+                            cls._name + ' ' + cls._synchronize_type)
+                res.append(cls_info)
+        return res
+
+    @api.onchange('method')
+    def onchange_method(self):
+        if self.method:
+            if 'import' in self.method:
+                self.method_type = 'import'
+            elif 'export' in self.method:
+                self.method_type = 'export'
+
+    @api.model
+    def _run(self, domain=None):
+        if not domain:
+            domain = []
+        tasks = self.env['external.file.task'].search(domain)
+        tasks.run()
+
+    @api.one
+    def run(self):
+        for cls in itersubclasses(AbstractTask):
+            if cls._synchronize_type and \
+                    cls._key + '_' + cls._synchronize_type == self.method:
+                method_class = cls
+        config = {
+            'host': self.location_id.address,
+            'user': self.location_id.login,
+            'pwd': self.location_id.password,
+            'port': self.location_id.port,
+            'allow_dir_creation': False,
+            'file_name': self.filename,
+            'path': self.filepath,
+            'attachment_ids': self.attachment_ids,
+            'task': self,
+            'move_path': self.move_path,
+            'after_import': self.after_import,
+            'md5_check': self.md5_check,
+            }
+        conn = method_class(self.env, config)
+        conn.run()

external_file_location/task_view.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+
+        <record id="view_task_form" model="ir.ui.view">
+            <field name="model">external.file.task</field>
+            <field name="arch" type="xml">
+                <form string="Tasks" version="7.0">
+                    <header>
+                        <button name="run" type="object" string="Run" icon="gtk-execute"/>
+                    </header>
+                    <sheet>
+                        <field name="method_type" invisible="1"/>
+                        <group col="4">
+                            <div class="oe_title" style="width: 390px;" colspan="4">
+                                <label class="oe_edit_only" for="name" string="Name"/>
+                                <h1><field name="name" class="oe_inline"/></h1>
+                            </div>
+                            <field name="method" colspan="2"/>
+                            <span colspan="2"/>
+                            <field name="filename" colspan="4" attrs="{'invisible':[('method_type','!=','import')], 'required':[('method_type', '=', 'import')]}"/>
+                            <field name="filepath" colspan="4" />
+                            <field name="after_import" colspan="2" attrs="{'invisible':[('method_type','!=','import')]}"/>
+                            <field name="move_path" colspan="2" attrs="{'invisible':['|', ('after_import','!=','move'), ('method_type','!=','import')]}"/>
+                            <field name="md5_check" colspan="2" attrs="{'invisible':[('method_type','!=','import')]}"/>
+                        </group>
+                    </sheet>
+                </form>
+            </field>
+        </record>
+
+        <record id="view_task_tree" model="ir.ui.view">
+            <field name="model">external.file.task</field>
+            <field name="arch" type="xml">
+                <tree string="Tasks" >
+                    <field name="name" select="1"/>
+                    <field name="method"/>
+                    <field name="filename"/>
+                    <field name="filepath"/>
+                    <button name="run" type="object" string="Run" icon="gtk-execute"/>
+                </tree>
+            </field>
+        </record>
+
+    </data>
+</openerp>

external_file_location/tasks/__init__.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from . import abstract_fs
+from . import ftp
+from . import sftp
+from . import filestore

external_file_location/tasks/abstract_fs.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
+#    @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+from ..abstract_task import AbstractTask
+import logging
+import os
+_logger = logging.getLogger(__name__)
+
+
+class AbstractFSTask(AbstractTask):
+
+    _name = None
+    _key = None
+    _synchronize_type = None
+    _default_port = None
+
+    def __init__(self, env, config):
+        self.env = env
+        self.host = config.get('host', '')
+        self.user = config.get('user', '')
+        self.pwd = config.get('pwd', '')
+        self.port = config.get('port', '')
+        self.allow_dir_creation = config.get('allow_dir_creation', '')
+        self.file_name = config.get('file_name', '')
+        self.path = config.get('path') or '.'
+        self.move_path = config.get('move_path', '')
+        self.after_import = config.get('after_import', False)
+        self.attachment_ids = config.get('attachment_ids', False)
+        self.task = config.get('task', False)
+        self.ext_hash = False
+        self.md5_check = config.get('md5_check', False)
+
+    def _handle_new_source(self, fs_conn, download_directory, file_name,
+                           move_directory):
+        """open and read given file into create_file method,
+           move file if move_directory is given"""
+        with fs_conn.open(self._source_name(download_directory, file_name),
+                          "rb") as fileobj:
+            data = fileobj.read()
+        return self.create_file(file_name, data)
+
+    def _source_name(self, download_directory, file_name):
+        """helper to get the full name"""
+        return os.path.join(download_directory, file_name)
+
+    def _move_file(self, fs_conn, source, target):
+        """Moves a file on the server"""
+        _logger.info('Moving file %s %s' % (source, target))
+        fs_conn.rename(source, target)
+        if self.md5_check:
+            fs_conn.rename(source + '.md5', target + '.md5')
+
+    def _delete_file(self, fs_conn, source):
+        """Deletes a file from the server"""
+        _logger.info('Deleting file %s' % source)
+        fs_conn.remove(source)
+        if self.md5_check:
+            fs_conn.remove(source + '.md5')
+
+    def _get_hash(self, file_name, fs_conn):
+        hash_file_name = file_name + '.md5'
+        with fs_conn.open(hash_file_name, 'rb') as f:
+            return f.read().rstrip('\r\n')
+
+    def _get_files(self, conn, path):
+        process_files = []
+        files_list = conn.listdir(path)
+        for file in files_list:
+            if file == self.file_name:
+                source_name = self._source_name(self.path, self.file_name)
+                process_files.append((file, source_name))
+        return process_files
+
+    def _process_file(self, conn, file_to_process):
+            if self.md5_check:
+                self.ext_hash = self._get_hash(file_to_process[1], conn)
+            att_id = self._handle_new_source(
+                conn,
+                self.path,
+                self.file_name,
+                self.move_path)
+
+            # Move/delete files only after all files have been processed.
+            if self.after_import == 'delete':
+                self._delete_file(conn, file_to_process[1])
+            elif self.after_import == 'move':
+                if not conn.exists(self.move_path):
+                    conn.makedir(self.move_path)
+                self._move_file(
+                    conn,
+                    file_to_process[1],
+                    self._source_name(self.move_path, file_to_process[0]))
+            return att_id
+
+    def _handle_existing_target(self, fs_conn, target_name, filedata):
+        raise Exception("%s already exists" % target_name)
+
+    def _handle_new_target(self, fs_conn, target_name, filedata):
+        try:
+            with fs_conn.open(target_name, mode='wb') as fileobj:
+                fileobj.write(filedata)
+                _logger.info('wrote %s, size %d', target_name, len(filedata))
+            self.attachment_id.state = 'done'
+            self.attachment_id.state_message = ''
+        except IOError:
+            self.attachment_id.state = 'failed'
+            self.attachment_id.state_message = (
+                'The directory doesn\'t exist or had insufficient rights')
+
+    def _target_name(self, fs_conn, upload_directory, filename):
+        return os.path.join(upload_directory, filename)
+
+    def _upload_file(self, conn, host, port, user, pwd,
+                     path, filename, filedata):
+        upload_directory = path or '.'
+        target_name = self._target_name(conn,
+                                        upload_directory,
+                                        filename)
+        if conn.isfile(target_name):
+            self._handle_existing_target(conn, target_name, filedata)
+        else:
+            self._handle_new_target(conn, target_name, filedata)

external_file_location/tasks/filestore.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
+#    @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+from .abstract_fs import AbstractFSTask
+from base64 import b64decode
+from fs import osfs
+import logging
+_logger = logging.getLogger(__name__)
+
+
+class FileStoreTask(AbstractFSTask):
+
+    _key = 'filestore'
+    _name = 'File Store'
+    _synchronize_type = None
+    _default_port = None
+    _hide_login = True
+    _hide_password = True
+    _hide_port = True
+
+
+class FileStoreImportTask(FileStoreTask):
+
+    _synchronize_type = 'import'
+
+    def run(self):
+        att_ids = []
+        with osfs.OSFS(self.host) as fs_conn:
+            files_to_process = self._get_files(fs_conn, self.path)
+            for file_to_process in files_to_process:
+                att_ids.append(self._process_file(fs_conn, file_to_process))
+        return att_ids
+
+
+class FileStoreExportTask(FileStoreTask):
+
+    _synchronize_type = 'export'
+
+    def run(self, async=True):
+        for attachment in self.attachment_ids:
+            if attachment.state in ('pending', 'failed'):
+                self.attachment_id = attachment
+                with osfs.OSFS(self.host) as fs_conn:
+                    self._upload_file(fs_conn,
+                                      self.host,
+                                      self.port,
+                                      self.user,
+                                      self.pwd,
+                                      self.path,
+                                      attachment.datas_fname,
+                                      b64decode(attachment.datas))

external_file_location/tasks/ftp.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
+#    @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+from .abstract_fs import AbstractFSTask
+from base64 import b64decode
+from fs import ftpfs
+import logging
+_logger = logging.getLogger(__name__)
+
+
+class FtpTask(AbstractFSTask):
+
+    _key = 'ftp'
+    _name = 'FTP'
+    _synchronize_type = None
+    _default_port = 21
+    _hide_login = False
+    _hide_password = False
+    _hide_port = False
+
+
+class FtpImportTask(FtpTask):
+
+    _synchronize_type = 'import'
+
+    def run(self):
+        att_ids = []
+        with ftpfs.FTPFS(self.host, self.user, self.pwd,
+                         port=self.port) as ftp_conn:
+            files_to_process = self._get_files(ftp_conn, self.path)
+            for file_to_process in files_to_process:
+                att_ids.append(self._process_file(ftp_conn, file_to_process))
+        return att_ids
+
+class FtpExportTask(FtpTask):
+
+    _synchronize_type = 'export'
+
+    def run(self, async=True):
+        import ipdb; ipdb.set_trace()
+        for attachment in self.attachment_ids:
+            if attachment.state in ('pending', 'failed'):
+                self.attachment_id = attachment
+                with ftpfs.FTPFS(self.host, self.user, self.pwd,
+                                 port=self.port) as ftp_conn:
+                    self._upload_file(ftp_conn, self.host, self.port,
+                                      self.user, self.pwd, self.path,
+                                      attachment.datas_fname,
+                                      b64decode(attachment.datas))

external_file_location/tasks/sftp.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
+#    @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+from .abstract_fs import AbstractFSTask
+from base64 import b64decode
+from fs import sftpfs
+import logging
+_logger = logging.getLogger(__name__)
+
+
+class SftpTask(AbstractFSTask):
+
+    _key = 'sftp'
+    _name = 'SFTP'
+    _synchronize_type = None
+    _default_port = 22
+    _hide_login = False
+    _hide_password = False
+    _hide_port = False
+
+
+class SftpImportTask(SftpTask):
+
+    _synchronize_type = 'import'
+
+    def run(self):
+        connection_string = "{}:{}".format(self.host, self.port)
+        root = "/home/{}".format(self.user)
+        att_ids = []
+        with sftpfs.SFTPFS(connection=connection_string,
+                           root_path=root,
+                           username=self.user,
+                           password=self.pwd) as sftp_conn:
+            files_to_process = self._get_files(sftp_conn, self.path)
+            for file_to_process in files_to_process:
+                att_ids.append(self._process_file(sftp_conn, file_to_process))
+        return att_ids
+
+
+class SftpExportTask(SftpTask):
+
+    _synchronize_type = 'export'
+
+    def run(self, async=True):
+        for attachment in self.attachment_ids:
+            if attachment.state in ('pending', 'failed'):
+                self.attachment_id = attachment
+                connection_string = "{}:{}".format(self.host, self.port)
+                with sftpfs.SFTPFS(connection=connection_string,
+                                   username=self.user,
+                                   password=self.pwd) as sftp_conn:
+                    datas = b64decode(attachment.datas)
+                    self._upload_file(sftp_conn, self.host, self.port,
+                                      self.user, self.pwd, self.path,
+                                      attachment.datas_fname, datas)

external_file_location/tests/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from . import mock_server
+from . import test_sftp

external_file_location/tests/mock_server.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import mock
+from contextlib import contextmanager
+from collections import defaultdict
+
+
+class MultiResponse(dict):
+    pass
+
+
+class ConnMock(object):
+
+    def __init__(self, response):
+        self.response = response
+        self._calls = []
+        self.call_count = defaultdict(int)
+
+    def __getattribute__(self, method):
+        if method not in ('_calls', 'response', 'call_count'):
+            def callable(*args, **kwargs):
+                self._calls.append({
+                    'method': method,
+                    'args': args,
+                    'kwargs': kwargs,
+                })
+                call = self.response[method]
+                if isinstance(call, MultiResponse):
+                    call = call[self.call_count[method]]
+                    self.call_count[method] += 1
+                return call
+
+            return callable
+        else:
+            return super(ConnMock, self).__getattribute__(method)
+
+    def __call__(self, *args, **kwargs):
+        return self
+
+    def __enter__(self, *args, **kwargs):
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        pass
+
+    def __repr__(self, *args, **kwargs):
+        return self
+
+    def __getitem__(self, key):
+        return
+
+
+@contextmanager
+def server_mock(response):
+    with mock.patch('fs.sftpfs.SFTPFS', ConnMock(response)) as SFTPFS:
+        yield SFTPFS._calls

external_file_location/tests/test_sftp.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import openerp.tests.common as common
+from ..tasks.sftp import SftpImportTask
+from ..tasks.sftp import SftpExportTask
+from .mock_server import (server_mock)
+from .mock_server import MultiResponse
+from StringIO import StringIO
+from base64 import b64decode
+import hashlib
+
+
+class ContextualStringIO(StringIO):
+    """
+    snippet from http://bit.ly/1HfH6uW (stackoverflow)
+    """
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+        return False
+
+
+class TestNewSource(common.TransactionCase):
+    def setUp(self):
+        super(TestNewSource, self).setUp()
+        self.test_file = ContextualStringIO()
+        self.test_file.write('import')
+        self.test_file.seek(0)
+        self.config = \
+            {'file_name': 'testfile',
+             'user': 'test',
+             'password': 'test',
+             'host': 'test',
+             'port': 22,
+             'attachment_ids': self.env['ir.attachment.metadata'].search([])
+             }
+
+    def test_00_sftp_import(self):
+        with server_mock(
+            {'exists': True,
+             'makedir': True,
+             'open': self.test_file,
+             'listdir': ['testfile']
+             }):
+            task = SftpImportTask(self.env, self.config)
+            task.run()
+        search_file = self.env['ir.attachment.metadata'].search(
+            (('name', '=', 'testfile'),))
+        self.assertEqual(len(search_file), 1)
+        self.assertEqual(b64decode(search_file[0].datas), 'import')
+
+    def test_01_sftp_export(self):
+        with server_mock(
+            {'isfile': False,
+             'open': self.test_file,
+             }) as FakeSFTP:
+            task = SftpExportTask(self.env, self.config)
+            task.run()
+            if FakeSFTP:
+                self.assertEqual('open', FakeSFTP[-1]['method'])
+
+    def test_02_sftp_import_delete(self):
+        with server_mock(
+            {'exists': True,
+             'makedir': True,
+             'open': self.test_file,
+             'listdir': ['testfile'],
+             'remove': True
+             }) as FakeSFTP:
+            self.config.update({'after_import': 'delete'})
+            task = SftpImportTask(self.env, self.config)
+            task.run()
+            search_file = self.env['ir.attachment.metadata'].search(
+                (('name', '=', 'testfile'),))
+            self.assertEqual(len(search_file), 1)
+            self.assertEqual(b64decode(search_file[0].datas), 'import')
+            self.assertEqual('remove', FakeSFTP[-1]['method'])
+
+    def test_03_sftp_import_move(self):
+        with server_mock(
+            {'exists': True,
+             'makedir': True,
+             'open': self.test_file,
+             'listdir': ['testfile'],
+             'rename': True
+             }) as FakeSFTP:
+            self.config.update({'after_import': 'move', 'move_path': '/home'})
+            task = SftpImportTask(self.env, self.config)
+            task.run()
+            search_file = self.env['ir.attachment.metadata'].search(
+                (('name', '=', 'testfile'),))
+            self.assertEqual(len(search_file), 1)
+            self.assertEqual(b64decode(search_file[0].datas), 'import')
+            self.assertEqual('rename', FakeSFTP[-1]['method'])
+
+    def test_04_sftp_import_md5(self):
+        md5_file = ContextualStringIO()
+        md5_file.write(hashlib.md5('import').hexdigest())
+        md5_file.seek(0)
+        with server_mock(
+            {'exists': True,
+             'makedir': True,
+             'open': MultiResponse({
+                 1: self.test_file,
+                 0: md5_file
+             }),
+             'listdir': ['testfile', 'testfile.md5'],
+             }) as FakeSFTP:
+            self.config.update({'md5_check': True})
+            task = SftpImportTask(self.env, self.config)
+            task.run()
+            search_file = self.env['ir.attachment.metadata'].search(
+                (('name', '=', 'testfile'),))
+            self.assertEqual(len(search_file), 1)
+            self.assertEqual(b64decode(search_file[0].datas), 'import')
+            self.assertEqual('open', FakeSFTP[-1]['method'])
+            self.assertEqual('open', FakeSFTP[1]['method'])
+            self.assertEqual(('./testfile.md5', 'rb'), FakeSFTP[1]['args'])