From 5063b5cc81f52d617780af60e52683dcc85ddf8c Mon Sep 17 00:00:00 2001 From: Dave Lasley Date: Mon, 22 Aug 2016 18:09:25 -0700 Subject: [PATCH] [MIG] auto_backup: Migrate to v9 * Add self.ensure_ones * Add test coverage --- auto_backup/README.rst | 20 +- .../{__openerp__.py => __manifest__.py} | 19 +- auto_backup/data/backup_data.yml | 28 --- auto_backup/data/ir_cron.xml | 18 ++ auto_backup/data/mail_message_subtype.xml | 19 ++ auto_backup/models/db_backup.py | 10 +- auto_backup/tests/__init__.py | 2 +- auto_backup/tests/test_auto_backup.py | 28 --- auto_backup/tests/test_db_backup.py | 232 ++++++++++++++++++ auto_backup/view/db_backup_view.xml | 9 +- 10 files changed, 307 insertions(+), 78 deletions(-) rename auto_backup/{__openerp__.py => __manifest__.py} (67%) delete mode 100644 auto_backup/data/backup_data.yml create mode 100644 auto_backup/data/ir_cron.xml create mode 100644 auto_backup/data/mail_message_subtype.xml delete mode 100644 auto_backup/tests/test_auto_backup.py create mode 100644 auto_backup/tests/test_db_backup.py diff --git a/auto_backup/README.rst b/auto_backup/README.rst index 7579fce45..e3957fe7b 100644 --- a/auto_backup/README.rst +++ b/auto_backup/README.rst @@ -70,15 +70,24 @@ manually execute the selected processes. .. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas :alt: Try me on Runbot - :target: https://runbot.odoo-community.org/runbot/149/8.0 + :target: https://runbot.odoo-community.org/runbot/149/10.0 + +Known issues / Roadmap +====================== + +* On larger databases, it is possible that backups will die due to Odoo server + settings. In order to circumvent this without frivolously changing settings, + you need to run the backup from outside of the main Odoo instance. How to do + this is outlined in `this blog post + `_. Bug Tracker =========== -Bugs are tracked on `GitHub Issues `_. -In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed feedback -`here `_. +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smashing it by providing a detailed and welcomed feedback. Credits ======= @@ -89,6 +98,7 @@ Contributors * Yenthe Van Ginneken * Alessio Gerace * Jairo Llopis +* Dave Lasley Maintainer ---------- diff --git a/auto_backup/__openerp__.py b/auto_backup/__manifest__.py similarity index 67% rename from auto_backup/__openerp__.py rename to auto_backup/__manifest__.py index 69979550a..7141e9372 100644 --- a/auto_backup/__openerp__.py +++ b/auto_backup/__manifest__.py @@ -7,19 +7,24 @@ { "name": "Database Auto-Backup", "summary": "Backups database", - "version": "8.0.1.0.1", + "version": "10.0.1.0.0", "author": ( - "VanRoey.be - Yenthe Van Ginneken, Agile Business Group," - " Grupo ESOC Ingeniería de Servicios," - " Odoo Community Association (OCA)" + "Yenthe Van Ginneken, " + "Agile Business Group, " + "Grupo ESOC Ingeniería de Servicios, " + "LasLabs, " + "Odoo Community Association (OCA)" ), 'license': "AGPL-3", "website": "http://www.vanroey.be/applications/bedrijfsbeheer/odoo", "category": "Tools", - "depends": ['email_template'], - "demo": [], + "depends": [ + 'base_setup', + 'mail', + ], "data": [ - "data/backup_data.yml", + "data/ir_cron.xml", + "data/mail_message_subtype.xml", "security/ir.model.access.csv", "view/db_backup_view.xml", ], diff --git a/auto_backup/data/backup_data.yml b/auto_backup/data/backup_data.yml deleted file mode 100644 index 3b6b4bc29..000000000 --- a/auto_backup/data/backup_data.yml +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). - -# Cron job -- !record {model: ir.cron, id: ir_cron_backupscheduler0}: - name: Backup scheduler - user_id: base.user_root - interval_number: 1 - interval_type: days - numbercall: -1 - nextcall: !eval - (datetime.now() + timedelta(days=1)).strftime("%Y-%m-%d 02:00:00") - model: db.backup - function: action_backup_all - -# New message subtypes -- !record {model: mail.message.subtype, id: success}: - name: Backup successful - res_model: db.backup - default: False - description: Database backup succeeded. - -- !record {model: mail.message.subtype, id: failure}: - name: Backup failed - res_model: db.backup - default: True - description: Database backup failed. diff --git a/auto_backup/data/ir_cron.xml b/auto_backup/data/ir_cron.xml new file mode 100644 index 000000000..6b62e364b --- /dev/null +++ b/auto_backup/data/ir_cron.xml @@ -0,0 +1,18 @@ + + + + + + Backup Scheduler + + 1 + days + -1 + + db.backup + action_backup_all + + + diff --git a/auto_backup/data/mail_message_subtype.xml b/auto_backup/data/mail_message_subtype.xml new file mode 100644 index 000000000..2dd820f97 --- /dev/null +++ b/auto_backup/data/mail_message_subtype.xml @@ -0,0 +1,19 @@ + + + + + + Backup Successful + Database backup succeeded. + db.backup + + + + + Backup Failed + Database backup failed. + db.backup + + + + diff --git a/auto_backup/models/db_backup.py b/auto_backup/models/db_backup.py index 28653d95d..4e30602aa 100644 --- a/auto_backup/models/db_backup.py +++ b/auto_backup/models/db_backup.py @@ -10,13 +10,13 @@ import traceback from contextlib import contextmanager from datetime import datetime, timedelta from glob import iglob -from openerp import exceptions, models, fields, api, _, tools -from openerp.service import db +from odoo import exceptions, models, fields, api, _, tools +from odoo.service import db import logging _logger = logging.getLogger(__name__) try: import pysftp -except ImportError: +except ImportError: # pragma: no cover _logger.debug('Cannot import pysftp') @@ -107,8 +107,8 @@ class DbBackup(models.Model): rec.name = "sftp://%s@%s:%d%s" % ( rec.sftp_user, rec.sftp_host, rec.sftp_port, rec.folder) - @api.constrains("folder", "method") @api.multi + @api.constrains("folder", "method") def _check_folder(self): """Do not use the filestore or you will backup your backups.""" for s in self: @@ -235,6 +235,7 @@ class DbBackup(models.Model): @contextmanager def cleanup_log(self): """Log a possible cleanup failure.""" + self.ensure_one() try: _logger.info("Starting cleanup process after database backup: %s", self.name) @@ -263,6 +264,7 @@ class DbBackup(models.Model): @api.multi def sftp_connection(self): """Return a new SFTP connection with found parameters.""" + self.ensure_one() params = { "host": self.sftp_host, "username": self.sftp_user, diff --git a/auto_backup/tests/__init__.py b/auto_backup/tests/__init__.py index ea27b998a..e803c71af 100644 --- a/auto_backup/tests/__init__.py +++ b/auto_backup/tests/__init__.py @@ -4,4 +4,4 @@ # © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -from . import test_auto_backup +from . import test_db_backup diff --git a/auto_backup/tests/test_auto_backup.py b/auto_backup/tests/test_auto_backup.py deleted file mode 100644 index dac490aba..000000000 --- a/auto_backup/tests/test_auto_backup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# © 2015 Agile Business Group -# © 2015 Alessio Gerace -# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). - -import os -from datetime import datetime -from openerp.tests import common - - -class TestsAutoBackup(common.TransactionCase): - - def setUp(self): - super(TestsAutoBackup, self).setUp() - self.abk = self.env["db.backup"].create( - { - 'name': u'Têst backup', - } - ) - - def test_local(self): - """A local database is backed up.""" - filename = self.abk.filename(datetime.now()) - self.abk.action_backup() - generated_backup = [f for f in os.listdir(self.abk.folder) - if f >= filename] - self.assertEqual(len(generated_backup), 1) diff --git a/auto_backup/tests/test_db_backup.py b/auto_backup/tests/test_db_backup.py new file mode 100644 index 000000000..91305a0aa --- /dev/null +++ b/auto_backup/tests/test_db_backup.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +# © 2015 Agile Business Group +# © 2015 Alessio Gerace +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# Copyright 2016 LasLabs Inc. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import os +import mock + +from datetime import datetime +from contextlib import contextmanager + +from odoo.tests import common +from odoo import exceptions, tools + +try: + import pysftp +except ImportError: + pass + + +model = 'odoo.addons.auto_backup.models.db_backup' + + +class TestConnectionException(pysftp.ConnectionException): + def __init__(self): + super(TestConnectionException, self).__init__('test', 'test') + + +class TestDbBackup(common.TransactionCase): + + def setUp(self): + super(TestDbBackup, self).setUp() + self.Model = self.env["db.backup"] + + @contextmanager + def mock_assets(self): + """ It provides mocked core assets """ + self.path_join_val = '/this/is/a/path' + with mock.patch('%s.db' % model) as db: + with mock.patch('%s.os' % model) as os: + with mock.patch('%s.shutil' % model) as shutil: + os.path.join.return_value = self.path_join_val + yield { + 'db': db, + 'os': os, + 'shutil': shutil, + } + + @contextmanager + def patch_filtered_sftp(self, record, mocks=None): + """ It patches filtered record and provides a mock """ + if mocks is None: + mocks = ['sftp_connection'] + mocks = {m: mock.DEFAULT for m in mocks} + with mock.patch.object(record, 'filtered') as filtered: + with mock.patch.object(record, 'backup_log'): + with mock.patch.multiple(record, **mocks): + filtered.side_effect = [], [record] + yield filtered + + def new_record(self, method='sftp'): + vals = { + 'name': u'Têst backup', + 'method': method, + } + if method == 'sftp': + vals.update({ + 'sftp_host': 'test_host', + 'sftp_port': '222', + 'sftp_user': 'tuser', + 'sftp_password': 'password', + 'folder': '/folder/', + }) + self.vals = vals + return self.Model.create(vals) + + def test_compute_name_sftp(self): + """ It should create proper SFTP URI """ + rec_id = self.new_record() + self.assertEqual( + 'sftp://%(user)s@%(host)s:%(port)s%(folder)s' % { + 'user': self.vals['sftp_user'], + 'host': self.vals['sftp_host'], + 'port': self.vals['sftp_port'], + 'folder': self.vals['folder'], + }, + rec_id.name, + ) + + def test_check_folder(self): + """ It should not allow recursive backups """ + rec_id = self.new_record('local') + with self.assertRaises(exceptions.ValidationError): + rec_id.write({ + 'folder': '%s/another/path' % tools.config.filestore( + self.env.cr.dbname + ), + }) + + @mock.patch('%s._' % model) + def test_action_sftp_test_connection_success(self, _): + """ It should raise connection succeeded warning """ + rec_id = self.new_record() + with mock.patch.object(rec_id, 'sftp_connection'): + with self.assertRaises(exceptions.Warning): + rec_id.action_sftp_test_connection() + _.assert_called_once_with("Connection Test Succeeded!") + + @mock.patch('%s._' % model) + def test_action_sftp_test_connection_fail(self, _): + """ It should raise connection fail warning """ + rec_id = self.new_record() + with mock.patch.object(rec_id, 'sftp_connection') as conn: + conn().__enter__.side_effect = TestConnectionException + with self.assertRaises(exceptions.Warning): + rec_id.action_sftp_test_connection() + _.assert_called_once_with("Connection Test Failed!") + + def test_action_backup_local(self): + """ It should backup local database """ + rec_id = self.new_record('local') + filename = rec_id.filename(datetime.now()) + rec_id.action_backup() + generated_backup = [f for f in os.listdir(rec_id.folder) + if f >= filename] + self.assertEqual(1, len(generated_backup)) + + def test_action_backup_sftp_mkdirs(self): + """ It should create remote dirs """ + rec_id = self.new_record() + with self.mock_assets(): + with self.patch_filtered_sftp(rec_id): + conn = rec_id.sftp_connection().__enter__() + rec_id.action_backup() + conn.makedirs.assert_called_once_with(rec_id.folder) + + def test_action_backup_sftp_mkdirs_conn_exception(self): + """ It should guard from ConnectionException on remote.mkdirs """ + rec_id = self.new_record() + with self.mock_assets(): + with self.patch_filtered_sftp(rec_id): + conn = rec_id.sftp_connection().__enter__() + conn.makedirs.side_effect = TestConnectionException + rec_id.action_backup() + # No error was raised, test pass + self.assertTrue(True) + + def test_action_backup_sftp_remote_open(self): + """ It should open remote file w/ proper args """ + rec_id = self.new_record() + with self.mock_assets() as assets: + with self.patch_filtered_sftp(rec_id): + conn = rec_id.sftp_connection().__enter__() + rec_id.action_backup() + conn.open.assert_called_once_with( + assets['os'].path.join(), + 'wb' + ) + + def test_action_backup_sftp_remote_open(self): + """ It should open remote file w/ proper args """ + rec_id = self.new_record() + with self.mock_assets() as assets: + with self.patch_filtered_sftp(rec_id): + conn = rec_id.sftp_connection().__enter__() + rec_id.action_backup() + conn.open.assert_called_once_with( + assets['os'].path.join(), + 'wb' + ) + + def test_action_backup_all_search(self): + """ It should search all records """ + rec_id = self.new_record() + with mock.patch.object(rec_id, 'search'): + rec_id.action_backup_all() + rec_id.search.assert_called_once_with([]) + + def test_action_backup_all_return(self): + """ It should return result of backup operation """ + rec_id = self.new_record() + with mock.patch.object(rec_id, 'search'): + res = rec_id.action_backup_all() + self.assertEqual( + rec_id.search().action_backup(), res + ) + + @mock.patch('%s.pysftp' % model) + def test_sftp_connection_init_passwd(self, pysftp): + """ It should initiate SFTP connection w/ proper args and pass """ + rec_id = self.new_record() + rec_id.sftp_connection() + pysftp.Connection.assert_called_once_with( + host=rec_id.sftp_host, + username=rec_id.sftp_user, + port=rec_id.sftp_port, + password=rec_id.sftp_password, + ) + + @mock.patch('%s.pysftp' % model) + def test_sftp_connection_init_key(self, pysftp): + """ It should initiate SFTP connection w/ proper args and key """ + rec_id = self.new_record() + rec_id.write({ + 'sftp_private_key': 'pkey', + 'sftp_password': 'pkeypass', + }) + rec_id.sftp_connection() + pysftp.Connection.assert_called_once_with( + host=rec_id.sftp_host, + username=rec_id.sftp_user, + port=rec_id.sftp_port, + private_key=rec_id.sftp_private_key, + private_key_pass=rec_id.sftp_password, + ) + + @mock.patch('%s.pysftp' % model) + def test_sftp_connection_return(self, pysftp): + """ It should return new sftp connection """ + rec_id = self.new_record() + res = rec_id.sftp_connection() + self.assertEqual( + pysftp.Connection(), res, + ) + + def test_filename(self): + """ It should not error and should return a .dump.zip file str """ + now = datetime.now() + res = self.Model.filename(now) + self.assertTrue(res.endswith(".dump.zip")) diff --git a/auto_backup/view/db_backup_view.xml b/auto_backup/view/db_backup_view.xml index d1f11e442..f36ff2361 100644 --- a/auto_backup/view/db_backup_view.xml +++ b/auto_backup/view/db_backup_view.xml @@ -1,6 +1,5 @@ - - + Automated Backups @@ -80,7 +79,7 @@ res_model="db.backup"/> @@ -104,5 +103,5 @@ db.backup client_action_multi - - + +